diff mbox series

[RFC,01/13] cxl: move header files for absolute references

Message ID 20240516081202.27023-2-alucerop@amd.com
State New, archived
Headers show
Series RFC: add Type2 device support | expand

Commit Message

Alejandro Lucero Palau May 16, 2024, 8:11 a.m. UTC
From: Alejandro Lucero <alucerop@amd.com>

CXL Type 2 devices imply specific vendor drivers binding to those
devices instead of generic ones offered by CXL core like the PCI driver.
Those drivers need to use CXL core functions and structs for
initialization, create memdevs and create CXL regions.

This patch avoids referencing those files based on relative paths from
inside the kernel sources tree.

Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
 drivers/cxl/acpi.c                      | 4 ++--
 drivers/cxl/core/cdat.c                 | 6 +++---
 drivers/cxl/core/hdm.c                  | 2 +-
 drivers/cxl/core/mbox.c                 | 6 +++---
 drivers/cxl/core/memdev.c               | 2 +-
 drivers/cxl/core/pci.c                  | 6 +++---
 drivers/cxl/core/pmem.c                 | 4 ++--
 drivers/cxl/core/pmu.c                  | 4 ++--
 drivers/cxl/core/port.c                 | 6 +++---
 drivers/cxl/core/region.c               | 4 ++--
 drivers/cxl/core/regs.c                 | 4 ++--
 drivers/cxl/core/suspend.c              | 2 +-
 drivers/cxl/core/trace.c                | 2 +-
 drivers/cxl/core/trace.h                | 4 ++--
 drivers/cxl/mem.c                       | 4 ++--
 drivers/cxl/pci.c                       | 6 +++---
 drivers/cxl/pmem.c                      | 4 ++--
 drivers/cxl/port.c                      | 4 ++--
 drivers/cxl/security.c                  | 4 ++--
 drivers/dax/cxl.c                       | 2 +-
 drivers/perf/cxl_pmu.c                  | 4 ++--
 {drivers/cxl => include/linux}/cxl.h    | 0
 {drivers/cxl => include/linux}/cxlmem.h | 2 +-
 {drivers/cxl => include/linux}/cxlpci.h | 0
 tools/testing/cxl/cxl_core_exports.c    | 2 +-
 tools/testing/cxl/mock_acpi.c           | 2 +-
 tools/testing/cxl/test/cxl.c            | 2 +-
 tools/testing/cxl/test/mem.c            | 2 +-
 tools/testing/cxl/test/mock.c           | 4 ++--
 tools/testing/cxl/test/mock.h           | 2 +-
 30 files changed, 50 insertions(+), 50 deletions(-)
 rename {drivers/cxl => include/linux}/cxl.h (100%)
 rename {drivers/cxl => include/linux}/cxlmem.h (99%)
 rename {drivers/cxl => include/linux}/cxlpci.h (100%)

Comments

Dan Williams June 12, 2024, 4:27 a.m. UTC | #1
alucerop@ wrote:
> From: Alejandro Lucero <alucerop@amd.com>
> 
> CXL Type 2 devices imply specific vendor drivers binding to those
> devices instead of generic ones offered by CXL core like the PCI driver.
> Those drivers need to use CXL core functions and structs for
> initialization, create memdevs and create CXL regions.
> 
> This patch avoids referencing those files based on relative paths from
> inside the kernel sources tree.
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
>  drivers/cxl/acpi.c                      | 4 ++--
>  drivers/cxl/core/cdat.c                 | 6 +++---
>  drivers/cxl/core/hdm.c                  | 2 +-
>  drivers/cxl/core/mbox.c                 | 6 +++---
>  drivers/cxl/core/memdev.c               | 2 +-
>  drivers/cxl/core/pci.c                  | 6 +++---
>  drivers/cxl/core/pmem.c                 | 4 ++--
>  drivers/cxl/core/pmu.c                  | 4 ++--
>  drivers/cxl/core/port.c                 | 6 +++---
>  drivers/cxl/core/region.c               | 4 ++--
>  drivers/cxl/core/regs.c                 | 4 ++--
>  drivers/cxl/core/suspend.c              | 2 +-
>  drivers/cxl/core/trace.c                | 2 +-
>  drivers/cxl/core/trace.h                | 4 ++--
>  drivers/cxl/mem.c                       | 4 ++--
>  drivers/cxl/pci.c                       | 6 +++---
>  drivers/cxl/pmem.c                      | 4 ++--
>  drivers/cxl/port.c                      | 4 ++--
>  drivers/cxl/security.c                  | 4 ++--
>  drivers/dax/cxl.c                       | 2 +-
>  drivers/perf/cxl_pmu.c                  | 4 ++--
>  {drivers/cxl => include/linux}/cxl.h    | 0
>  {drivers/cxl => include/linux}/cxlmem.h | 2 +-
>  {drivers/cxl => include/linux}/cxlpci.h | 0

So I don't like this approach, there are details that are private to the
CXL generic memory-expander use case, and some that are suitable for
CXL.mem and CXL.cache capabilities in other drivers. That distinct
subset should move to include/linux/. I.e. I want to see the incremental
conversion of what 3rd party drivers need compared to the generic
expander case, and consider when and where new shared infrastructure
needs to be refactored.
Christoph Hellwig June 12, 2024, 4:30 a.m. UTC | #2
On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
> alucerop@ wrote:
> > From: Alejandro Lucero <alucerop@amd.com>
> > 
> > CXL Type 2 devices imply specific vendor drivers binding to those
> > devices instead of generic ones offered by CXL core like the PCI driver.

No, it absolutelt does not.  There is no such thing as a vendor driver
in Linux.

> So I don't like this approach, there are details that are private to the
> CXL generic memory-expander use case, and some that are suitable for
> CXL.mem and CXL.cache capabilities in other drivers. That distinct
> subset should move to include/linux/. I.e. I want to see the incremental
> conversion of what 3rd party drivers need compared to the generic
> expander case, and consider when and where new shared infrastructure
> needs to be refactored.

And I'd much rather keep all these drivers in drivers/cxl/ anyway so
that we can keep a tight control over them.
Alejandro Lucero Palau June 12, 2024, 5:42 a.m. UTC | #3
On 6/12/24 05:27, Dan Williams wrote:
> alucerop@ wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> CXL Type 2 devices imply specific vendor drivers binding to those
>> devices instead of generic ones offered by CXL core like the PCI driver.
>> Those drivers need to use CXL core functions and structs for
>> initialization, create memdevs and create CXL regions.
>>
>> This patch avoids referencing those files based on relative paths from
>> inside the kernel sources tree.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>>   drivers/cxl/acpi.c                      | 4 ++--
>>   drivers/cxl/core/cdat.c                 | 6 +++---
>>   drivers/cxl/core/hdm.c                  | 2 +-
>>   drivers/cxl/core/mbox.c                 | 6 +++---
>>   drivers/cxl/core/memdev.c               | 2 +-
>>   drivers/cxl/core/pci.c                  | 6 +++---
>>   drivers/cxl/core/pmem.c                 | 4 ++--
>>   drivers/cxl/core/pmu.c                  | 4 ++--
>>   drivers/cxl/core/port.c                 | 6 +++---
>>   drivers/cxl/core/region.c               | 4 ++--
>>   drivers/cxl/core/regs.c                 | 4 ++--
>>   drivers/cxl/core/suspend.c              | 2 +-
>>   drivers/cxl/core/trace.c                | 2 +-
>>   drivers/cxl/core/trace.h                | 4 ++--
>>   drivers/cxl/mem.c                       | 4 ++--
>>   drivers/cxl/pci.c                       | 6 +++---
>>   drivers/cxl/pmem.c                      | 4 ++--
>>   drivers/cxl/port.c                      | 4 ++--
>>   drivers/cxl/security.c                  | 4 ++--
>>   drivers/dax/cxl.c                       | 2 +-
>>   drivers/perf/cxl_pmu.c                  | 4 ++--
>>   {drivers/cxl => include/linux}/cxl.h    | 0
>>   {drivers/cxl => include/linux}/cxlmem.h | 2 +-
>>   {drivers/cxl => include/linux}/cxlpci.h | 0
> So I don't like this approach, there are details that are private to the
> CXL generic memory-expander use case, and some that are suitable for
> CXL.mem and CXL.cache capabilities in other drivers. That distinct
> subset should move to include/linux/. I.e. I want to see the incremental
> conversion of what 3rd party drivers need compared to the generic
> expander case, and consider when and where new shared infrastructure
> needs to be refactored.
>

OK. It makes sense.

I'll work on that if there is a v2.

Thanks
Alejandro Lucero Palau June 12, 2024, 5:54 a.m. UTC | #4
On 6/12/24 05:30, Christoph Hellwig wrote:
> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
>> alucerop@ wrote:
>>> From: Alejandro Lucero <alucerop@amd.com>
>>>
>>> CXL Type 2 devices imply specific vendor drivers binding to those
>>> devices instead of generic ones offered by CXL core like the PCI driver.
> No, it absolutelt does not.  There is no such thing as a vendor driver
> in Linux.


Well, yes, I see your point, and it is absolutely correct.

It was a bad way of explaining the need of a driver supporting a 
specific hardware.


>> So I don't like this approach, there are details that are private to the
>> CXL generic memory-expander use case, and some that are suitable for
>> CXL.mem and CXL.cache capabilities in other drivers. That distinct
>> subset should move to include/linux/. I.e. I want to see the incremental
>> conversion of what 3rd party drivers need compared to the generic
>> expander case, and consider when and where new shared infrastructure
>> needs to be refactored.
> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
> that we can keep a tight control over them.
>

My initial thought was drivers for ethernet, gpus, or any other kind of 
standard device binding to the CXL/PCIe.io and doing the CXL 
initialization. Your comment seems to suggest other approach, which I 
can only relate to using auxbus for the device CXL.mem and CXL.cache, 
and  then being handled by a generic driver inside the CXL core. Is this 
what you are suggesting?

If so, I'm not against it, although it would require to study the 
implications.
Jonathan Cameron June 12, 2024, 10:07 a.m. UTC | #5
On Wed, 12 Jun 2024 06:54:13 +0100
Alejandro Lucero Palau <alucerop@amd.com> wrote:

> On 6/12/24 05:30, Christoph Hellwig wrote:
> > On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:  
> >> alucerop@ wrote:  
> >>> From: Alejandro Lucero <alucerop@amd.com>
> >>>
> >>> CXL Type 2 devices imply specific vendor drivers binding to those
> >>> devices instead of generic ones offered by CXL core like the PCI driver.  
> > No, it absolutelt does not.  There is no such thing as a vendor driver
> > in Linux.  
> 
> 
> Well, yes, I see your point, and it is absolutely correct.
> 
> It was a bad way of explaining the need of a driver supporting a 
> specific hardware.
> 
> 
> >> So I don't like this approach, there are details that are private to the
> >> CXL generic memory-expander use case, and some that are suitable for
> >> CXL.mem and CXL.cache capabilities in other drivers. That distinct
> >> subset should move to include/linux/. I.e. I want to see the incremental
> >> conversion of what 3rd party drivers need compared to the generic
> >> expander case, and consider when and where new shared infrastructure
> >> needs to be refactored.  
> > And I'd much rather keep all these drivers in drivers/cxl/ anyway so
> > that we can keep a tight control over them.
> >  
> 
> My initial thought was drivers for ethernet, gpus, or any other kind of 
> standard device binding to the CXL/PCIe.io and doing the CXL 
> initialization. Your comment seems to suggest other approach, which I 
> can only relate to using auxbus for the device CXL.mem and CXL.cache, 
> and  then being handled by a generic driver inside the CXL core. Is this 
> what you are suggesting?
> 
> If so, I'm not against it, although it would require to study the 
> implications.

I'd be very careful with that level of complexity.
So far I'm not seeing it as useful for this case (but I could be wrong)

Maybe just a case of well defined library code with only
opaque state etc being exposed by the CXL driver framework.

The bulk of any type2 driver needs to be in the appropriate subsystem
not drivers/cxl/  The non CXL parts will be more important to keep
aligned with appropriate subsystem than the CXL part. 

Jonathan

> 
>
Alejandro Lucero Palau June 12, 2024, 1:36 p.m. UTC | #6
On 6/12/24 11:07, Jonathan Cameron wrote:
> On Wed, 12 Jun 2024 06:54:13 +0100
> Alejandro Lucero Palau <alucerop@amd.com> wrote:
>
>> On 6/12/24 05:30, Christoph Hellwig wrote:
>>> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
>>>> alucerop@ wrote:
>>>>> From: Alejandro Lucero <alucerop@amd.com>
>>>>>
>>>>> CXL Type 2 devices imply specific vendor drivers binding to those
>>>>> devices instead of generic ones offered by CXL core like the PCI driver.
>>> No, it absolutelt does not.  There is no such thing as a vendor driver
>>> in Linux.
>>
>> Well, yes, I see your point, and it is absolutely correct.
>>
>> It was a bad way of explaining the need of a driver supporting a
>> specific hardware.
>>
>>
>>>> So I don't like this approach, there are details that are private to the
>>>> CXL generic memory-expander use case, and some that are suitable for
>>>> CXL.mem and CXL.cache capabilities in other drivers. That distinct
>>>> subset should move to include/linux/. I.e. I want to see the incremental
>>>> conversion of what 3rd party drivers need compared to the generic
>>>> expander case, and consider when and where new shared infrastructure
>>>> needs to be refactored.
>>> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
>>> that we can keep a tight control over them.
>>>   
>> My initial thought was drivers for ethernet, gpus, or any other kind of
>> standard device binding to the CXL/PCIe.io and doing the CXL
>> initialization. Your comment seems to suggest other approach, which I
>> can only relate to using auxbus for the device CXL.mem and CXL.cache,
>> and  then being handled by a generic driver inside the CXL core. Is this
>> what you are suggesting?
>>
>> If so, I'm not against it, although it would require to study the
>> implications.
> I'd be very careful with that level of complexity.
> So far I'm not seeing it as useful for this case (but I could be wrong)


I agree the complexity could not be justified.

Indeed I think the privacy of a CXL.mem, something I pointed out, will 
make things more complicated.


> Maybe just a case of well defined library code with only
> opaque state etc being exposed by the CXL driver framework.


+1


> The bulk of any type2 driver needs to be in the appropriate subsystem
> not drivers/cxl/  The non CXL parts will be more important to keep
> aligned with appropriate subsystem than the CXL part.


+1


> Jonathan
>
>>
Dan Williams June 12, 2024, 9:18 p.m. UTC | #7
Christoph Hellwig wrote:
> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
> > alucerop@ wrote:
> > > From: Alejandro Lucero <alucerop@amd.com>
> > > 
> > > CXL Type 2 devices imply specific vendor drivers binding to those
> > > devices instead of generic ones offered by CXL core like the PCI driver.
> 
> No, it absolutelt does not.  There is no such thing as a vendor driver
> in Linux.
> 
> > So I don't like this approach, there are details that are private to the
> > CXL generic memory-expander use case, and some that are suitable for
> > CXL.mem and CXL.cache capabilities in other drivers. That distinct
> > subset should move to include/linux/. I.e. I want to see the incremental
> > conversion of what 3rd party drivers need compared to the generic
> > expander case, and consider when and where new shared infrastructure
> > needs to be refactored.
> 
> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
> that we can keep a tight control over them.

Yeah, especially for the common pieces like CXL memory where the
endpoint driver is responsible for registering a 'struct cxl_memdev' and
the existing 'cxl_mem' driver just grows to accommodate device-memory
alongside host-only memory expansion.

For CXL accelerator use case that's where more specifics are needed
because CXL.cache operation gets entangled with IOMMU / DMA mapping
policy for ATS.
Alejandro Lucero Palau June 13, 2024, 11:45 a.m. UTC | #8
On 6/12/24 22:18, Dan Williams wrote:
> Christoph Hellwig wrote:
>> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
>>> alucerop@ wrote:
>>>> From: Alejandro Lucero <alucerop@amd.com>
>>>>
>>>> CXL Type 2 devices imply specific vendor drivers binding to those
>>>> devices instead of generic ones offered by CXL core like the PCI driver.
>> No, it absolutelt does not.  There is no such thing as a vendor driver
>> in Linux.
>>
>>> So I don't like this approach, there are details that are private to the
>>> CXL generic memory-expander use case, and some that are suitable for
>>> CXL.mem and CXL.cache capabilities in other drivers. That distinct
>>> subset should move to include/linux/. I.e. I want to see the incremental
>>> conversion of what 3rd party drivers need compared to the generic
>>> expander case, and consider when and where new shared infrastructure
>>> needs to be refactored.
>> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
>> that we can keep a tight control over them.
> Yeah, especially for the common pieces like CXL memory where the
> endpoint driver is responsible for registering a 'struct cxl_memdev' and
> the existing 'cxl_mem' driver just grows to accommodate device-memory
> alongside host-only memory expansion.
>
> For CXL accelerator use case that's where more specifics are needed
> because CXL.cache operation gets entangled with IOMMU / DMA mapping
> policy for ATS.

I think we all agree the central pieces should be in CXL core and other 
drivers requiring CXL setup going through such central management.

But this can be implemented as I thought and tried, and Jonathan has 
also suggested, that is an API with opaque structs, or through a more 
disjointed way from a client/driver perspective using an approach like 
auxbus. Maybe there is another approach for "keep all these drivers in 
drivers/cxl/ anyway so that we can keep a tight control over them" which 
needs to be more explicitly elaborated.

My plan is for a v2 removing the test/mock driver and adding the sfc 
driver as a client, but only if the original approach with the CXL core 
exporting functions is the approved one. If auxbus or other way is the 
way to go, it will be just a new patchset and requiring more time.
Dan Williams June 14, 2024, 1:22 a.m. UTC | #9
Alejandro Lucero Palau wrote:
[..]
> On 6/12/24 22:18, Dan Williams wrote:
> > Christoph Hellwig wrote:
> >> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
> >>> alucerop@ wrote:
> >>>> From: Alejandro Lucero <alucerop@amd.com>
> >>>>
> >>>> CXL Type 2 devices imply specific vendor drivers binding to those
> >>>> devices instead of generic ones offered by CXL core like the PCI driver.
> >> No, it absolutelt does not.  There is no such thing as a vendor driver
> >> in Linux.
> >>
> >>> So I don't like this approach, there are details that are private to the
> >>> CXL generic memory-expander use case, and some that are suitable for
> >>> CXL.mem and CXL.cache capabilities in other drivers. That distinct
> >>> subset should move to include/linux/. I.e. I want to see the incremental
> >>> conversion of what 3rd party drivers need compared to the generic
> >>> expander case, and consider when and where new shared infrastructure
> >>> needs to be refactored.
> >> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
> >> that we can keep a tight control over them.
> > Yeah, especially for the common pieces like CXL memory where the
> > endpoint driver is responsible for registering a 'struct cxl_memdev' and
> > the existing 'cxl_mem' driver just grows to accommodate device-memory
> > alongside host-only memory expansion.
> >
> > For CXL accelerator use case that's where more specifics are needed
> > because CXL.cache operation gets entangled with IOMMU / DMA mapping
> > policy for ATS.
> 
> I think we all agree the central pieces should be in CXL core and other 
> drivers requiring CXL setup going through such central management.
> 
> But this can be implemented as I thought and tried

Like I said earlier, start by minimizing the definitions to just what is
needed as that may trip over things that need to be refactored.

For example, we had a similar conversation about how to support the CXL
mailbox which appears to be showing up in more than just generic CXL
memory expanders. In that case the observation is that all the APIs that
currently assume a 'struct cxl_memdev_state *' probably need to be
refactored into something that takes a new 'struct cxl_mbox *'.

> and Jonathan has also suggested, that is an API with opaque structs,

Opaque structs is even nicer, but first step is "don't make the entirety
of the current definitions public" because that violates 'least
privilege' for many of these structures. 

> or through a more disjointed way from a client/driver perspective
> using an approach like auxbus.

What? No, I see no need for auxbus. "cxl" is already a bus.

> Maybe there is another approach for "keep all these drivers in
> drivers/cxl/ anyway so that we can keep a tight control over them"
> which needs to be more explicitly elaborated.

For now lets interpret that to mean that any driver that wants to
enumerate or provision a topology of CXL HDM decoders should be looking
to register a 'struct cxl_memdev' object which is driven by the cxl_mem
driver in drivers/cxl/. The client driver that registers that need not be
located in drivers/cxl/.

> My plan is for a v2 removing the test/mock driver and adding the sfc 
> driver as a client,

Oh, as in CONFIG_SFC? Can you say anything about the use case for
CXL.cache and CXL.mem so that we make sure the code architecture lines
are being drawn at the right level?

> but only if the original approach with the CXL core 
> exporting functions is the approved one.

Not sure what you mean here? Exporting common CXL core functions is how
the cxl_mem driver itself gets enabled. So there is no other path
besides refactoring the core to be useful outside of the general memory
expansion use case. This is analogous to how the PCI core provides
infrastructure for PCI drivers.

> If auxbus or other way is the way to go, it will be just a new
> patchset and requiring more time.

Mainline has the time to get it right.
Alejandro Lucero Palau June 14, 2024, 8:54 a.m. UTC | #10
On 6/14/24 02:22, Dan Williams wrote:
> Alejandro Lucero Palau wrote:
> [..]
>> On 6/12/24 22:18, Dan Williams wrote:
>>> Christoph Hellwig wrote:
>>>> On Tue, Jun 11, 2024 at 09:27:38PM -0700, Dan Williams wrote:
>>>>> alucerop@ wrote:
>>>>>> From: Alejandro Lucero <alucerop@amd.com>
>>>>>>
>>>>>> CXL Type 2 devices imply specific vendor drivers binding to those
>>>>>> devices instead of generic ones offered by CXL core like the PCI driver.
>>>> No, it absolutelt does not.  There is no such thing as a vendor driver
>>>> in Linux.
>>>>
>>>>> So I don't like this approach, there are details that are private to the
>>>>> CXL generic memory-expander use case, and some that are suitable for
>>>>> CXL.mem and CXL.cache capabilities in other drivers. That distinct
>>>>> subset should move to include/linux/. I.e. I want to see the incremental
>>>>> conversion of what 3rd party drivers need compared to the generic
>>>>> expander case, and consider when and where new shared infrastructure
>>>>> needs to be refactored.
>>>> And I'd much rather keep all these drivers in drivers/cxl/ anyway so
>>>> that we can keep a tight control over them.
>>> Yeah, especially for the common pieces like CXL memory where the
>>> endpoint driver is responsible for registering a 'struct cxl_memdev' and
>>> the existing 'cxl_mem' driver just grows to accommodate device-memory
>>> alongside host-only memory expansion.
>>>
>>> For CXL accelerator use case that's where more specifics are needed
>>> because CXL.cache operation gets entangled with IOMMU / DMA mapping
>>> policy for ATS.
>> I think we all agree the central pieces should be in CXL core and other
>> drivers requiring CXL setup going through such central management.
>>
>> But this can be implemented as I thought and tried
> Like I said earlier, start by minimizing the definitions to just what is
> needed as that may trip over things that need to be refactored.


Yes, I agree the headers treatment in the patch was too much coarse 
work. I'll work on that for v2.


> For example, we had a similar conversation about how to support the CXL
> mailbox which appears to be showing up in more than just generic CXL
> memory expanders. In that case the observation is that all the APIs that
> currently assume a 'struct cxl_memdev_state *' probably need to be
> refactored into something that takes a new 'struct cxl_mbox *'.
>
>> and Jonathan has also suggested, that is an API with opaque structs,
> Opaque structs is even nicer, but first step is "don't make the entirety
> of the current definitions public" because that violates 'least
> privilege' for many of these structures.
>
>> or through a more disjointed way from a client/driver perspective
>> using an approach like auxbus.
> What? No, I see no need for auxbus. "cxl" is already a bus.
>> Maybe there is another approach for "keep all these drivers in
>> drivers/cxl/ anyway so that we can keep a tight control over them"
>> which needs to be more explicitly elaborated.
> For now lets interpret that to mean that any driver that wants to
> enumerate or provision a topology of CXL HDM decoders should be looking
> to register a 'struct cxl_memdev' object which is driven by the cxl_mem
> driver in drivers/cxl/. The client driver that registers that need not be
> located in drivers/cxl/.


Great. This needed clarification after your previous email.


>> My plan is for a v2 removing the test/mock driver and adding the sfc
>> driver as a client,
> Oh, as in CONFIG_SFC? Can you say anything about the use case for
> CXL.cache and CXL.mem so that we make sure the code architecture lines
> are being drawn at the right level?


I did briefly comment on that in the cover-letter discussion exchange.

We got some hardware buffers now mapped from BAR regions which are 
written by the driver for low latency sending versus descriptor + packet 
through DMA.

The idea is those buffers being CXL.mem mappings now.

CXL.cache will be for receiving. It will not be part of the first 
products though.


>> but only if the original approach with the CXL core
>> exporting functions is the approved one.
> Not sure what you mean here? Exporting common CXL core functions is how
> the cxl_mem driver itself gets enabled. So there is no other path
> besides refactoring the core to be useful outside of the general memory
> expansion use case. This is analogous to how the PCI core provides
> infrastructure for PCI drivers.


OK. This is all related to that comment/suggestion for keeping all CXL 
capable drivers in the CXL core directory.

The only way I could envision such an approach was through a driver 
creating auxiliary devices for CXL.mem and CXL.cache which could bind to 
generic CXL drivers (with the appropriate support in current CXL core). 
Doable but probably too much complicated.


I think it is all clear now.

Thanks


>> If auxbus or other way is the way to go, it will be just a new
>> patchset and requiring more time.
> Mainline has the time to get it right.
diff mbox series

Patch

diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
index cb8c155a2c9b..f023898382bc 100644
--- a/drivers/cxl/acpi.c
+++ b/drivers/cxl/acpi.c
@@ -8,8 +8,8 @@ 
 #include <linux/pci.h>
 #include <linux/node.h>
 #include <asm/div64.h>
-#include "cxlpci.h"
-#include "cxl.h"
+#include <linux/cxlpci.h>
+#include <linux/cxl.h>
 
 #define CXL_RCRB_SIZE	SZ_8K
 
diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c
index bb83867d9fec..97ff1dfd63d6 100644
--- a/drivers/cxl/core/cdat.c
+++ b/drivers/cxl/core/cdat.c
@@ -5,10 +5,10 @@ 
 #include <linux/fw_table.h>
 #include <linux/node.h>
 #include <linux/overflow.h>
-#include "cxlpci.h"
-#include "cxlmem.h"
+#include <linux/cxlpci.h>
+#include <linux/cxlmem.h>
 #include "core.h"
-#include "cxl.h"
+#include <linux/cxl.h>
 #include "core.h"
 
 struct dsmas_entry {
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 7d97790b893d..47d9faf5897f 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -4,7 +4,7 @@ 
 #include <linux/device.h>
 #include <linux/delay.h>
 
-#include "cxlmem.h"
+#include <linux/cxlmem.h>
 #include "core.h"
 
 /**
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index f0f54aeccc87..d312b82f7f36 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -5,9 +5,9 @@ 
 #include <linux/ktime.h>
 #include <linux/mutex.h>
 #include <asm/unaligned.h>
-#include <cxlpci.h>
-#include <cxlmem.h>
-#include <cxl.h>
+#include <linux/cxlpci.h>
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 
 #include "core.h"
 #include "trace.h"
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index d4e259f3a7e9..07cd0b8b026f 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -7,7 +7,7 @@ 
 #include <linux/slab.h>
 #include <linux/idr.h>
 #include <linux/pci.h>
-#include <cxlmem.h>
+#include <linux/cxlmem.h>
 #include "trace.h"
 #include "core.h"
 
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 0df09bd79408..a494a03b4a83 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -7,9 +7,9 @@ 
 #include <linux/pci.h>
 #include <linux/pci-doe.h>
 #include <linux/aer.h>
-#include <cxlpci.h>
-#include <cxlmem.h>
-#include <cxl.h>
+#include <linux/cxlpci.h>
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 #include "core.h"
 #include "trace.h"
 
diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
index e69625a8d6a1..d1a5f8d9cf91 100644
--- a/drivers/cxl/core/pmem.c
+++ b/drivers/cxl/core/pmem.c
@@ -3,8 +3,8 @@ 
 #include <linux/device.h>
 #include <linux/slab.h>
 #include <linux/idr.h>
-#include <cxlmem.h>
-#include <cxl.h>
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 #include "core.h"
 
 /**
diff --git a/drivers/cxl/core/pmu.c b/drivers/cxl/core/pmu.c
index 5d8e06b0ba6e..aa02ea582184 100644
--- a/drivers/cxl/core/pmu.c
+++ b/drivers/cxl/core/pmu.c
@@ -4,9 +4,9 @@ 
 #include <linux/device.h>
 #include <linux/slab.h>
 #include <linux/idr.h>
-#include <cxlmem.h>
+#include <linux/cxlmem.h>
 #include <pmu.h>
-#include <cxl.h>
+#include <linux/cxl.h>
 #include "core.h"
 
 static void cxl_pmu_release(struct device *dev)
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 762783bb091a..fdd76306260d 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -11,9 +11,9 @@ 
 #include <linux/slab.h>
 #include <linux/idr.h>
 #include <linux/node.h>
-#include <cxlmem.h>
-#include <cxlpci.h>
-#include <cxl.h>
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
+#include <linux/cxl.h>
 #include "core.h"
 
 /**
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 5c186e0a39b9..70e86a7c241d 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -9,8 +9,8 @@ 
 #include <linux/uuid.h>
 #include <linux/sort.h>
 #include <linux/idr.h>
-#include <cxlmem.h>
-#include <cxl.h>
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 #include "core.h"
 
 /**
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 3c42f984eeaf..fd165e718cf2 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -4,8 +4,8 @@ 
 #include <linux/device.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
-#include <cxlmem.h>
-#include <cxlpci.h>
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
 #include <pmu.h>
 
 #include "core.h"
diff --git a/drivers/cxl/core/suspend.c b/drivers/cxl/core/suspend.c
index a5984d96ea1d..117b3398ee56 100644
--- a/drivers/cxl/core/suspend.c
+++ b/drivers/cxl/core/suspend.c
@@ -2,7 +2,7 @@ 
 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
 #include <linux/atomic.h>
 #include <linux/export.h>
-#include "cxlmem.h"
+#include <linux/cxlmem.h>
 
 static atomic_t mem_active;
 
diff --git a/drivers/cxl/core/trace.c b/drivers/cxl/core/trace.c
index d0403dc3c8ab..1e6ba0de237a 100644
--- a/drivers/cxl/core/trace.c
+++ b/drivers/cxl/core/trace.c
@@ -1,7 +1,7 @@ 
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
 
-#include <cxl.h>
+#include <linux/cxl.h>
 #include "core.h"
 
 #define CREATE_TRACE_POINTS
diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h
index e5f13260fc52..e3b1156a01ad 100644
--- a/drivers/cxl/core/trace.h
+++ b/drivers/cxl/core/trace.h
@@ -10,8 +10,8 @@ 
 #include <linux/pci.h>
 #include <asm-generic/unaligned.h>
 
-#include <cxl.h>
-#include <cxlmem.h>
+#include <linux/cxl.h>
+#include <linux/cxlmem.h>
 #include "core.h"
 
 #define CXL_RAS_UC_CACHE_DATA_PARITY	BIT(0)
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 0c79d9ce877c..6dc2bf1e2b1a 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -5,8 +5,8 @@ 
 #include <linux/module.h>
 #include <linux/pci.h>
 
-#include "cxlmem.h"
-#include "cxlpci.h"
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
 
 /**
  * DOC: cxl mem
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 2ff361e756d6..ccde33ac9c1c 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -11,9 +11,9 @@ 
 #include <linux/pci.h>
 #include <linux/aer.h>
 #include <linux/io.h>
-#include "cxlmem.h"
-#include "cxlpci.h"
-#include "cxl.h"
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
+#include <linux/cxl.h>
 #include "pmu.h"
 
 /**
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 7cb8994f8809..deadc5ffc2c2 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -8,8 +8,8 @@ 
 #include <linux/async.h>
 #include <linux/slab.h>
 #include <linux/nd.h>
-#include "cxlmem.h"
-#include "cxl.h"
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 
 extern const struct nvdimm_security_ops *cxl_security_ops;
 
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index 97c21566677a..928ebff774ce 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -4,8 +4,8 @@ 
 #include <linux/module.h>
 #include <linux/slab.h>
 
-#include "cxlmem.h"
-#include "cxlpci.h"
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
 
 /**
  * DOC: cxl port
diff --git a/drivers/cxl/security.c b/drivers/cxl/security.c
index 21856a3f408e..129b36928a05 100644
--- a/drivers/cxl/security.c
+++ b/drivers/cxl/security.c
@@ -6,8 +6,8 @@ 
 #include <linux/async.h>
 #include <linux/slab.h>
 #include <linux/memregion.h>
-#include "cxlmem.h"
-#include "cxl.h"
+#include <linux/cxlmem.h>
+#include <linux/cxl.h>
 
 static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm,
 						 enum nvdimm_passphrase_type ptype)
diff --git a/drivers/dax/cxl.c b/drivers/dax/cxl.c
index c696837ab23c..89a5c8eb666e 100644
--- a/drivers/dax/cxl.c
+++ b/drivers/dax/cxl.c
@@ -3,7 +3,7 @@ 
 #include <linux/module.h>
 #include <linux/dax.h>
 
-#include "../cxl/cxl.h"
+#include <linux/cxl.h>
 #include "bus.h"
 
 static int cxl_dax_region_probe(struct device *dev)
diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c
index 308c9969642e..9b93ba215bdb 100644
--- a/drivers/perf/cxl_pmu.c
+++ b/drivers/perf/cxl_pmu.c
@@ -20,8 +20,8 @@ 
 #include <linux/bug.h>
 #include <linux/pci.h>
 
-#include "../cxl/cxlpci.h"
-#include "../cxl/cxl.h"
+#include <linux/cxlpci.h>
+#include <linux/cxl.h>
 #include "../cxl/pmu.h"
 
 #define CXL_PMU_CAP_REG			0x0
diff --git a/drivers/cxl/cxl.h b/include/linux/cxl.h
similarity index 100%
rename from drivers/cxl/cxl.h
rename to include/linux/cxl.h
diff --git a/drivers/cxl/cxlmem.h b/include/linux/cxlmem.h
similarity index 99%
rename from drivers/cxl/cxlmem.h
rename to include/linux/cxlmem.h
index 36cee9c30ceb..0d26a45a4af2 100644
--- a/drivers/cxl/cxlmem.h
+++ b/include/linux/cxlmem.h
@@ -8,7 +8,7 @@ 
 #include <linux/rcuwait.h>
 #include <linux/cxl-event.h>
 #include <linux/node.h>
-#include "cxl.h"
+#include <linux/cxl.h>
 
 /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */
 #define CXLMDEV_STATUS_OFFSET 0x0
diff --git a/drivers/cxl/cxlpci.h b/include/linux/cxlpci.h
similarity index 100%
rename from drivers/cxl/cxlpci.h
rename to include/linux/cxlpci.h
diff --git a/tools/testing/cxl/cxl_core_exports.c b/tools/testing/cxl/cxl_core_exports.c
index 077e6883921d..737b49ed3e46 100644
--- a/tools/testing/cxl/cxl_core_exports.c
+++ b/tools/testing/cxl/cxl_core_exports.c
@@ -1,7 +1,7 @@ 
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
 
-#include "cxl.h"
+#include <linux/cxl.h>
 
 /* Exporting of cxl_core symbols that are only used by cxl_test */
 EXPORT_SYMBOL_NS_GPL(cxl_num_decoders_committed, CXL);
diff --git a/tools/testing/cxl/mock_acpi.c b/tools/testing/cxl/mock_acpi.c
index 55813de26d46..4e440a9c0cb2 100644
--- a/tools/testing/cxl/mock_acpi.c
+++ b/tools/testing/cxl/mock_acpi.c
@@ -4,7 +4,7 @@ 
 #include <linux/platform_device.h>
 #include <linux/device.h>
 #include <linux/acpi.h>
-#include <cxl.h>
+#include <linux/cxl.h>
 #include "test/mock.h"
 
 struct acpi_device *to_cxl_host_bridge(struct device *host, struct device *dev)
diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
index 61c69297e797..848c42c2c158 100644
--- a/tools/testing/cxl/test/cxl.c
+++ b/tools/testing/cxl/test/cxl.c
@@ -8,7 +8,7 @@ 
 #include <linux/acpi.h>
 #include <linux/pci.h>
 #include <linux/mm.h>
-#include <cxlmem.h>
+#include <linux/cxlmem.h>
 
 #include "../watermark.h"
 #include "mock.h"
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index 35ee41e435ab..dcddb6affc0d 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -9,7 +9,7 @@ 
 #include <linux/bits.h>
 #include <asm/unaligned.h>
 #include <crypto/sha2.h>
-#include <cxlmem.h>
+#include <linux/cxlmem.h>
 
 #include "trace.h"
 
diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c
index 6f737941dc0e..a1366a24677f 100644
--- a/tools/testing/cxl/test/mock.c
+++ b/tools/testing/cxl/test/mock.c
@@ -7,8 +7,8 @@ 
 #include <linux/export.h>
 #include <linux/acpi.h>
 #include <linux/pci.h>
-#include <cxlmem.h>
-#include <cxlpci.h>
+#include <linux/cxlmem.h>
+#include <linux/cxlpci.h>
 #include "mock.h"
 
 static LIST_HEAD(mock);
diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h
index d1b0271d2822..bf7ec147ea80 100644
--- a/tools/testing/cxl/test/mock.h
+++ b/tools/testing/cxl/test/mock.h
@@ -2,7 +2,7 @@ 
 
 #include <linux/list.h>
 #include <linux/acpi.h>
-#include <cxl.h>
+#include <linux/cxl.h>
 
 struct cxl_mock_ops {
 	struct list_head list;