diff mbox series

[rdma-next,v2,01/11] RDMA: Add EFA related definitions

Message ID 1550763193-14128-2-git-send-email-galpress@amazon.com (mailing list archive)
State Superseded
Headers show
Series RDMA/efa: Elastic Fabric Adapter (EFA) driver | expand

Commit Message

Gal Pressman Feb. 21, 2019, 3:33 p.m. UTC
Add EFA node, transport and protocol types to core code.
In addition, EFA driver ID is added to the IOCTL interface uapi.

Signed-off-by: Gal Pressman <galpress@amazon.com>
---
 drivers/infiniband/core/sysfs.c          | 1 +
 drivers/infiniband/core/verbs.c          | 2 ++
 include/rdma/ib_verbs.h                  | 8 ++++++--
 include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
 4 files changed, 10 insertions(+), 2 deletions(-)

Comments

Steve Wise Feb. 26, 2019, 5:47 p.m. UTC | #1
On 2/21/2019 9:33 AM, Gal Pressman wrote:
> Add EFA node, transport and protocol types to core code.
> In addition, EFA driver ID is added to the IOCTL interface uapi.
>
> Signed-off-by: Gal Pressman <galpress@amazon.com>
> ---
>  drivers/infiniband/core/sysfs.c          | 1 +
>  drivers/infiniband/core/verbs.c          | 2 ++
>  include/rdma/ib_verbs.h                  | 8 ++++++--
>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>  4 files changed, 10 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
> index 9b6a065bdfa5..f7fac4c81559 100644
> --- a/drivers/infiniband/core/sysfs.c
> +++ b/drivers/infiniband/core/sysfs.c
> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> index 5a5e83f5f0fc..fc6f748a1933 100644
> --- a/drivers/infiniband/core/verbs.c
> +++ b/drivers/infiniband/core/verbs.c
> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
>  		return RDMA_TRANSPORT_USNIC_UDP;
>  	if (node_type == RDMA_NODE_RNIC)
>  		return RDMA_TRANSPORT_IWARP;
> +	if (node_type == RDMA_NODE_EFA)
> +		return RDMA_TRANSPORT_EFA;
>  
>  	return RDMA_TRANSPORT_IB;
>  }
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index 225cb76d469f..55534aba22bf 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -108,6 +108,7 @@ enum rdma_node_type {
>  	RDMA_NODE_RNIC,
>  	RDMA_NODE_USNIC,
>  	RDMA_NODE_USNIC_UDP,
> +	RDMA_NODE_EFA,
>  };
>  
>  enum {
> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>  	RDMA_TRANSPORT_IB,
>  	RDMA_TRANSPORT_IWARP,
>  	RDMA_TRANSPORT_USNIC,
> -	RDMA_TRANSPORT_USNIC_UDP
> +	RDMA_TRANSPORT_USNIC_UDP,
> +	RDMA_TRANSPORT_EFA,
>  };
>  
>  enum rdma_protocol_type {
>  	RDMA_PROTOCOL_IB,
>  	RDMA_PROTOCOL_IBOE,
>  	RDMA_PROTOCOL_IWARP,
> -	RDMA_PROTOCOL_USNIC_UDP
> +	RDMA_PROTOCOL_USNIC_UDP,
> +	RDMA_PROTOCOL_EFA,
>  };
>  
>  __attribute_const__ enum rdma_transport_type
> @@ -543,6 +546,7 @@ static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
>  #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
>  #define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
>  #define RDMA_CORE_CAP_PROT_USNIC        0x02000000
> +#define RDMA_CORE_CAP_PROT_EFA          0x04000000
>  
>  #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
>  					| RDMA_CORE_CAP_PROT_ROCE     \
> diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
> index 06c34d99be85..26213f49f5c8 100644
> --- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
> +++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
> @@ -102,6 +102,7 @@ enum rdma_driver_id {
>  	RDMA_DRIVER_RXE,
>  	RDMA_DRIVER_HFI1,
>  	RDMA_DRIVER_QIB,
> +	RDMA_DRIVER_EFA,
>  };
>  
>  #endif


Looks fine.

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Steve Wise Feb. 26, 2019, 5:56 p.m. UTC | #2
On 2/26/2019 11:47 AM, Steve Wise wrote:
> On 2/21/2019 9:33 AM, Gal Pressman wrote:
>> Add EFA node, transport and protocol types to core code.
>> In addition, EFA driver ID is added to the IOCTL interface uapi.
>>
>> Signed-off-by: Gal Pressman <galpress@amazon.com>
>> ---
>>  drivers/infiniband/core/sysfs.c          | 1 +
>>  drivers/infiniband/core/verbs.c          | 2 ++
>>  include/rdma/ib_verbs.h                  | 8 ++++++--
>>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>>  4 files changed, 10 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
>> index 9b6a065bdfa5..f7fac4c81559 100644
>> --- a/drivers/infiniband/core/sysfs.c
>> +++ b/drivers/infiniband/core/sysfs.c
>> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
>>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
>>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
>> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
>>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
>>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
>>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
>> index 5a5e83f5f0fc..fc6f748a1933 100644
>> --- a/drivers/infiniband/core/verbs.c
>> +++ b/drivers/infiniband/core/verbs.c
>> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
>>  		return RDMA_TRANSPORT_USNIC_UDP;
>>  	if (node_type == RDMA_NODE_RNIC)
>>  		return RDMA_TRANSPORT_IWARP;
>> +	if (node_type == RDMA_NODE_EFA)
>> +		return RDMA_TRANSPORT_EFA;
>>  
>>  	return RDMA_TRANSPORT_IB;
>>  }
>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
>> index 225cb76d469f..55534aba22bf 100644
>> --- a/include/rdma/ib_verbs.h
>> +++ b/include/rdma/ib_verbs.h
>> @@ -108,6 +108,7 @@ enum rdma_node_type {
>>  	RDMA_NODE_RNIC,
>>  	RDMA_NODE_USNIC,
>>  	RDMA_NODE_USNIC_UDP,
>> +	RDMA_NODE_EFA,
>>  };
>>  
>>  enum {
>> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>>  	RDMA_TRANSPORT_IB,
>>  	RDMA_TRANSPORT_IWARP,
>>  	RDMA_TRANSPORT_USNIC,
>> -	RDMA_TRANSPORT_USNIC_UDP
>> +	RDMA_TRANSPORT_USNIC_UDP,
>> +	RDMA_TRANSPORT_EFA,
>>  };
>>  
>>  enum rdma_protocol_type {
>>  	RDMA_PROTOCOL_IB,
>>  	RDMA_PROTOCOL_IBOE,
>>  	RDMA_PROTOCOL_IWARP,
>> -	RDMA_PROTOCOL_USNIC_UDP
>> +	RDMA_PROTOCOL_USNIC_UDP,
>> +	RDMA_PROTOCOL_EFA,
>>  };
>>  
>>  __attribute_const__ enum rdma_transport_type
>> @@ -543,6 +546,7 @@ static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
>>  #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
>>  #define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
>>  #define RDMA_CORE_CAP_PROT_USNIC        0x02000000
>> +#define RDMA_CORE_CAP_PROT_EFA          0x04000000
>>  
>>  #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
>>  					| RDMA_CORE_CAP_PROT_ROCE     \
>> diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
>> index 06c34d99be85..26213f49f5c8 100644
>> --- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
>> +++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
>> @@ -102,6 +102,7 @@ enum rdma_driver_id {
>>  	RDMA_DRIVER_RXE,
>>  	RDMA_DRIVER_HFI1,
>>  	RDMA_DRIVER_QIB,
>> +	RDMA_DRIVER_EFA,
>>  };
>>  
>>  #endif
>
> Looks fine.
>
> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
>
Oops - correction:

Reviewed-by: Steve Wise <swise@opengridcomputing.com>
Leon Romanovsky Feb. 26, 2019, 7:20 p.m. UTC | #3
On Thu, Feb 21, 2019 at 05:33:03PM +0200, Gal Pressman wrote:
> Add EFA node, transport and protocol types to core code.
> In addition, EFA driver ID is added to the IOCTL interface uapi.
>
> Signed-off-by: Gal Pressman <galpress@amazon.com>
> ---
>  drivers/infiniband/core/sysfs.c          | 1 +
>  drivers/infiniband/core/verbs.c          | 2 ++
>  include/rdma/ib_verbs.h                  | 8 ++++++--
>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>  4 files changed, 10 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
> index 9b6a065bdfa5..f7fac4c81559 100644
> --- a/drivers/infiniband/core/sysfs.c
> +++ b/drivers/infiniband/core/sysfs.c
> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> index 5a5e83f5f0fc..fc6f748a1933 100644
> --- a/drivers/infiniband/core/verbs.c
> +++ b/drivers/infiniband/core/verbs.c
> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
>  		return RDMA_TRANSPORT_USNIC_UDP;
>  	if (node_type == RDMA_NODE_RNIC)
>  		return RDMA_TRANSPORT_IWARP;
> +	if (node_type == RDMA_NODE_EFA)
> +		return RDMA_TRANSPORT_EFA;
>
>  	return RDMA_TRANSPORT_IB;
>  }
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index 225cb76d469f..55534aba22bf 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -108,6 +108,7 @@ enum rdma_node_type {
>  	RDMA_NODE_RNIC,
>  	RDMA_NODE_USNIC,
>  	RDMA_NODE_USNIC_UDP,
> +	RDMA_NODE_EFA,
>  };
>
>  enum {
> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>  	RDMA_TRANSPORT_IB,
>  	RDMA_TRANSPORT_IWARP,
>  	RDMA_TRANSPORT_USNIC,
> -	RDMA_TRANSPORT_USNIC_UDP
> +	RDMA_TRANSPORT_USNIC_UDP,
> +	RDMA_TRANSPORT_EFA,
>  };
>
>  enum rdma_protocol_type {
>  	RDMA_PROTOCOL_IB,
>  	RDMA_PROTOCOL_IBOE,
>  	RDMA_PROTOCOL_IWARP,
> -	RDMA_PROTOCOL_USNIC_UDP
> +	RDMA_PROTOCOL_USNIC_UDP,
> +	RDMA_PROTOCOL_EFA,
>  };

Everything above is backed by specification. I disagree that we need
to declare new transport, protocol and node every time some large company
decides to reinvent the wheel. Especially if this company refuses
provide specification grade documentation.

Thanks
Gal Pressman Feb. 27, 2019, 7:45 a.m. UTC | #4
On 26-Feb-19 21:20, Leon Romanovsky wrote:
> On Thu, Feb 21, 2019 at 05:33:03PM +0200, Gal Pressman wrote:
>> Add EFA node, transport and protocol types to core code.
>> In addition, EFA driver ID is added to the IOCTL interface uapi.
>>
>> Signed-off-by: Gal Pressman <galpress@amazon.com>
>> ---
>>  drivers/infiniband/core/sysfs.c          | 1 +
>>  drivers/infiniband/core/verbs.c          | 2 ++
>>  include/rdma/ib_verbs.h                  | 8 ++++++--
>>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>>  4 files changed, 10 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
>> index 9b6a065bdfa5..f7fac4c81559 100644
>> --- a/drivers/infiniband/core/sysfs.c
>> +++ b/drivers/infiniband/core/sysfs.c
>> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
>>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
>>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
>> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
>>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
>>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
>>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
>> index 5a5e83f5f0fc..fc6f748a1933 100644
>> --- a/drivers/infiniband/core/verbs.c
>> +++ b/drivers/infiniband/core/verbs.c
>> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
>>  		return RDMA_TRANSPORT_USNIC_UDP;
>>  	if (node_type == RDMA_NODE_RNIC)
>>  		return RDMA_TRANSPORT_IWARP;
>> +	if (node_type == RDMA_NODE_EFA)
>> +		return RDMA_TRANSPORT_EFA;
>>
>>  	return RDMA_TRANSPORT_IB;
>>  }
>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
>> index 225cb76d469f..55534aba22bf 100644
>> --- a/include/rdma/ib_verbs.h
>> +++ b/include/rdma/ib_verbs.h
>> @@ -108,6 +108,7 @@ enum rdma_node_type {
>>  	RDMA_NODE_RNIC,
>>  	RDMA_NODE_USNIC,
>>  	RDMA_NODE_USNIC_UDP,
>> +	RDMA_NODE_EFA,
>>  };
>>
>>  enum {
>> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>>  	RDMA_TRANSPORT_IB,
>>  	RDMA_TRANSPORT_IWARP,
>>  	RDMA_TRANSPORT_USNIC,
>> -	RDMA_TRANSPORT_USNIC_UDP
>> +	RDMA_TRANSPORT_USNIC_UDP,
>> +	RDMA_TRANSPORT_EFA,
>>  };
>>
>>  enum rdma_protocol_type {
>>  	RDMA_PROTOCOL_IB,
>>  	RDMA_PROTOCOL_IBOE,
>>  	RDMA_PROTOCOL_IWARP,
>> -	RDMA_PROTOCOL_USNIC_UDP
>> +	RDMA_PROTOCOL_USNIC_UDP,
>> +	RDMA_PROTOCOL_EFA,
>>  };
> 
> Everything above is backed by specification. I disagree that we need
> to declare new transport, protocol and node every time some large company
> decides to reinvent the wheel. Especially if this company refuses
> provide specification grade documentation.

EFA is not an IB_CA node/IB transport/IB protocol.
We have two options:
1. Add EFA definitions
2. Add 'unspecified' definitions

Both will not be backed up by the spec, at least with option #1 we'll have more
information than an ambiguous unspecified.
Gal Pressman Feb. 27, 2019, 8:01 a.m. UTC | #5
On 26-Feb-19 19:56, Steve Wise wrote:
> 
> On 2/26/2019 11:47 AM, Steve Wise wrote:
>> On 2/21/2019 9:33 AM, Gal Pressman wrote:
>>> Add EFA node, transport and protocol types to core code.
>>> In addition, EFA driver ID is added to the IOCTL interface uapi.
>>>
>>> Signed-off-by: Gal Pressman <galpress@amazon.com>
>>> ---
>>>  drivers/infiniband/core/sysfs.c          | 1 +
>>>  drivers/infiniband/core/verbs.c          | 2 ++
>>>  include/rdma/ib_verbs.h                  | 8 ++++++--
>>>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>>>  4 files changed, 10 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
>>> index 9b6a065bdfa5..f7fac4c81559 100644
>>> --- a/drivers/infiniband/core/sysfs.c
>>> +++ b/drivers/infiniband/core/sysfs.c
>>> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>>>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
>>>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
>>>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
>>> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
>>>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
>>>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
>>>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
>>> index 5a5e83f5f0fc..fc6f748a1933 100644
>>> --- a/drivers/infiniband/core/verbs.c
>>> +++ b/drivers/infiniband/core/verbs.c
>>> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
>>>  		return RDMA_TRANSPORT_USNIC_UDP;
>>>  	if (node_type == RDMA_NODE_RNIC)
>>>  		return RDMA_TRANSPORT_IWARP;
>>> +	if (node_type == RDMA_NODE_EFA)
>>> +		return RDMA_TRANSPORT_EFA;
>>>  
>>>  	return RDMA_TRANSPORT_IB;
>>>  }
>>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
>>> index 225cb76d469f..55534aba22bf 100644
>>> --- a/include/rdma/ib_verbs.h
>>> +++ b/include/rdma/ib_verbs.h
>>> @@ -108,6 +108,7 @@ enum rdma_node_type {
>>>  	RDMA_NODE_RNIC,
>>>  	RDMA_NODE_USNIC,
>>>  	RDMA_NODE_USNIC_UDP,
>>> +	RDMA_NODE_EFA,
>>>  };
>>>  
>>>  enum {
>>> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>>>  	RDMA_TRANSPORT_IB,
>>>  	RDMA_TRANSPORT_IWARP,
>>>  	RDMA_TRANSPORT_USNIC,
>>> -	RDMA_TRANSPORT_USNIC_UDP
>>> +	RDMA_TRANSPORT_USNIC_UDP,
>>> +	RDMA_TRANSPORT_EFA,
>>>  };
>>>  
>>>  enum rdma_protocol_type {
>>>  	RDMA_PROTOCOL_IB,
>>>  	RDMA_PROTOCOL_IBOE,
>>>  	RDMA_PROTOCOL_IWARP,
>>> -	RDMA_PROTOCOL_USNIC_UDP
>>> +	RDMA_PROTOCOL_USNIC_UDP,
>>> +	RDMA_PROTOCOL_EFA,
>>>  };
>>>  
>>>  __attribute_const__ enum rdma_transport_type
>>> @@ -543,6 +546,7 @@ static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
>>>  #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
>>>  #define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
>>>  #define RDMA_CORE_CAP_PROT_USNIC        0x02000000
>>> +#define RDMA_CORE_CAP_PROT_EFA          0x04000000
>>>  
>>>  #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
>>>  					| RDMA_CORE_CAP_PROT_ROCE     \
>>> diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
>>> index 06c34d99be85..26213f49f5c8 100644
>>> --- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
>>> +++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
>>> @@ -102,6 +102,7 @@ enum rdma_driver_id {
>>>  	RDMA_DRIVER_RXE,
>>>  	RDMA_DRIVER_HFI1,
>>>  	RDMA_DRIVER_QIB,
>>> +	RDMA_DRIVER_EFA,
>>>  };
>>>  
>>>  #endif
>>
>> Looks fine.
>>
>> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
>>
> Oops - correction:
> 
> Reviewed-by: Steve Wise <swise@opengridcomputing.com>

Thanks Steve!
Leon Romanovsky Feb. 27, 2019, 8:02 a.m. UTC | #6
On Wed, Feb 27, 2019 at 09:45:06AM +0200, Gal Pressman wrote:
> On 26-Feb-19 21:20, Leon Romanovsky wrote:
> > On Thu, Feb 21, 2019 at 05:33:03PM +0200, Gal Pressman wrote:
> >> Add EFA node, transport and protocol types to core code.
> >> In addition, EFA driver ID is added to the IOCTL interface uapi.
> >>
> >> Signed-off-by: Gal Pressman <galpress@amazon.com>
> >> ---
> >>  drivers/infiniband/core/sysfs.c          | 1 +
> >>  drivers/infiniband/core/verbs.c          | 2 ++
> >>  include/rdma/ib_verbs.h                  | 8 ++++++--
> >>  include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
> >>  4 files changed, 10 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
> >> index 9b6a065bdfa5..f7fac4c81559 100644
> >> --- a/drivers/infiniband/core/sysfs.c
> >> +++ b/drivers/infiniband/core/sysfs.c
> >> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
> >>  	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
> >>  	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
> >>  	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
> >> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
> >>  	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
> >>  	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
> >>  	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
> >> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> >> index 5a5e83f5f0fc..fc6f748a1933 100644
> >> --- a/drivers/infiniband/core/verbs.c
> >> +++ b/drivers/infiniband/core/verbs.c
> >> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
> >>  		return RDMA_TRANSPORT_USNIC_UDP;
> >>  	if (node_type == RDMA_NODE_RNIC)
> >>  		return RDMA_TRANSPORT_IWARP;
> >> +	if (node_type == RDMA_NODE_EFA)
> >> +		return RDMA_TRANSPORT_EFA;
> >>
> >>  	return RDMA_TRANSPORT_IB;
> >>  }
> >> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> >> index 225cb76d469f..55534aba22bf 100644
> >> --- a/include/rdma/ib_verbs.h
> >> +++ b/include/rdma/ib_verbs.h
> >> @@ -108,6 +108,7 @@ enum rdma_node_type {
> >>  	RDMA_NODE_RNIC,
> >>  	RDMA_NODE_USNIC,
> >>  	RDMA_NODE_USNIC_UDP,
> >> +	RDMA_NODE_EFA,
> >>  };
> >>
> >>  enum {
> >> @@ -119,14 +120,16 @@ enum rdma_transport_type {
> >>  	RDMA_TRANSPORT_IB,
> >>  	RDMA_TRANSPORT_IWARP,
> >>  	RDMA_TRANSPORT_USNIC,
> >> -	RDMA_TRANSPORT_USNIC_UDP
> >> +	RDMA_TRANSPORT_USNIC_UDP,
> >> +	RDMA_TRANSPORT_EFA,
> >>  };
> >>
> >>  enum rdma_protocol_type {
> >>  	RDMA_PROTOCOL_IB,
> >>  	RDMA_PROTOCOL_IBOE,
> >>  	RDMA_PROTOCOL_IWARP,
> >> -	RDMA_PROTOCOL_USNIC_UDP
> >> +	RDMA_PROTOCOL_USNIC_UDP,
> >> +	RDMA_PROTOCOL_EFA,
> >>  };
> >
> > Everything above is backed by specification. I disagree that we need
> > to declare new transport, protocol and node every time some large company
> > decides to reinvent the wheel. Especially if this company refuses
> > provide specification grade documentation.
>
> EFA is not an IB_CA node/IB transport/IB protocol.
> We have two options:
> 1. Add EFA definitions
> 2. Add 'unspecified' definitions
>
> Both will not be backed up by the spec, at least with option #1 we'll have more
> information than an ambiguous unspecified.

Excellent options for out-of-tree driver.

Those fields are important for interopability, both EFA and "unspecified"
doesn't qualify for that. So there is third option - do standardization.

Thanks
Shiraz Saleem Feb. 27, 2019, 10:08 p.m. UTC | #7
>Subject: [PATCH rdma-next v2 01/11] RDMA: Add EFA related definitions
>
>Add EFA node, transport and protocol types to core code.
>In addition, EFA driver ID is added to the IOCTL interface uapi.
>
>Signed-off-by: Gal Pressman <galpress@amazon.com>
>---
> drivers/infiniband/core/sysfs.c          | 1 +
> drivers/infiniband/core/verbs.c          | 2 ++
> include/rdma/ib_verbs.h                  | 8 ++++++--
> include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
> 4 files changed, 10 insertions(+), 2 deletions(-)
>
>diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index
>9b6a065bdfa5..f7fac4c81559 100644
>--- a/drivers/infiniband/core/sysfs.c
>+++ b/drivers/infiniband/core/sysfs.c
>@@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
> 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev-
>>node_type);
> 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev-
>>node_type);
> 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n",
>dev->node_type);
>+	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev-
>>node_type);
> 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev-
>>node_type);
> 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev-
>>node_type);
> 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index
>5a5e83f5f0fc..fc6f748a1933 100644
>--- a/drivers/infiniband/core/verbs.c
>+++ b/drivers/infiniband/core/verbs.c
>@@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type
>node_type)
> 		return RDMA_TRANSPORT_USNIC_UDP;
> 	if (node_type == RDMA_NODE_RNIC)
> 		return RDMA_TRANSPORT_IWARP;
>+	if (node_type == RDMA_NODE_EFA)
>+		return RDMA_TRANSPORT_EFA;
>
> 	return RDMA_TRANSPORT_IB;
> }
>diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
>225cb76d469f..55534aba22bf 100644
>--- a/include/rdma/ib_verbs.h
>+++ b/include/rdma/ib_verbs.h
>@@ -108,6 +108,7 @@ enum rdma_node_type {
> 	RDMA_NODE_RNIC,
> 	RDMA_NODE_USNIC,
> 	RDMA_NODE_USNIC_UDP,
>+	RDMA_NODE_EFA,
> };
>
> enum {
>@@ -119,14 +120,16 @@ enum rdma_transport_type {
> 	RDMA_TRANSPORT_IB,
> 	RDMA_TRANSPORT_IWARP,
> 	RDMA_TRANSPORT_USNIC,
>-	RDMA_TRANSPORT_USNIC_UDP
>+	RDMA_TRANSPORT_USNIC_UDP,
>+	RDMA_TRANSPORT_EFA,
> };
>
> enum rdma_protocol_type {
> 	RDMA_PROTOCOL_IB,
> 	RDMA_PROTOCOL_IBOE,
> 	RDMA_PROTOCOL_IWARP,
>-	RDMA_PROTOCOL_USNIC_UDP
>+	RDMA_PROTOCOL_USNIC_UDP,
>+	RDMA_PROTOCOL_EFA,
> };
>

Link layer is Ethernet? And layer 3, network type?

Reviewed-by: Shiraz Saleem <shiraz.saleem@intel.com>
Gal Pressman Feb. 28, 2019, 7:31 a.m. UTC | #8
On 28-Feb-19 00:08, Saleem, Shiraz wrote:
>> Subject: [PATCH rdma-next v2 01/11] RDMA: Add EFA related definitions
>>
>> Add EFA node, transport and protocol types to core code.
>> In addition, EFA driver ID is added to the IOCTL interface uapi.
>>
>> Signed-off-by: Gal Pressman <galpress@amazon.com>
>> ---
>> drivers/infiniband/core/sysfs.c          | 1 +
>> drivers/infiniband/core/verbs.c          | 2 ++
>> include/rdma/ib_verbs.h                  | 8 ++++++--
>> include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>> 4 files changed, 10 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index
>> 9b6a065bdfa5..f7fac4c81559 100644
>> --- a/drivers/infiniband/core/sysfs.c
>> +++ b/drivers/infiniband/core/sysfs.c
>> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>> 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev-
>>> node_type);
>> 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev-
>>> node_type);
>> 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n",
>> dev->node_type);
>> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev-
>>> node_type);
>> 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev-
>>> node_type);
>> 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev-
>>> node_type);
>> 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index
>> 5a5e83f5f0fc..fc6f748a1933 100644
>> --- a/drivers/infiniband/core/verbs.c
>> +++ b/drivers/infiniband/core/verbs.c
>> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type
>> node_type)
>> 		return RDMA_TRANSPORT_USNIC_UDP;
>> 	if (node_type == RDMA_NODE_RNIC)
>> 		return RDMA_TRANSPORT_IWARP;
>> +	if (node_type == RDMA_NODE_EFA)
>> +		return RDMA_TRANSPORT_EFA;
>>
>> 	return RDMA_TRANSPORT_IB;
>> }
>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
>> 225cb76d469f..55534aba22bf 100644
>> --- a/include/rdma/ib_verbs.h
>> +++ b/include/rdma/ib_verbs.h
>> @@ -108,6 +108,7 @@ enum rdma_node_type {
>> 	RDMA_NODE_RNIC,
>> 	RDMA_NODE_USNIC,
>> 	RDMA_NODE_USNIC_UDP,
>> +	RDMA_NODE_EFA,
>> };
>>
>> enum {
>> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>> 	RDMA_TRANSPORT_IB,
>> 	RDMA_TRANSPORT_IWARP,
>> 	RDMA_TRANSPORT_USNIC,
>> -	RDMA_TRANSPORT_USNIC_UDP
>> +	RDMA_TRANSPORT_USNIC_UDP,
>> +	RDMA_TRANSPORT_EFA,
>> };
>>
>> enum rdma_protocol_type {
>> 	RDMA_PROTOCOL_IB,
>> 	RDMA_PROTOCOL_IBOE,
>> 	RDMA_PROTOCOL_IWARP,
>> -	RDMA_PROTOCOL_USNIC_UDP
>> +	RDMA_PROTOCOL_USNIC_UDP,
>> +	RDMA_PROTOCOL_EFA,
>> };
>>
> 
> Link layer is Ethernet? And layer 3, network type?

Our link layer is Ethernet, but the wire protocol is not the same as RoCE so it
was decided to use unspecified link layer.
The wire protocol is tightly coupled to the wire protocol for EC2’s VPC software
defined network (EFA is not designed to be used independently of EC2’s VPC data
plane).

> 
> Reviewed-by: Shiraz Saleem <shiraz.saleem@intel.com>

Thanks Shiraz!
Jason Gunthorpe March 5, 2019, 1:32 a.m. UTC | #9
On Thu, Feb 28, 2019 at 09:31:07AM +0200, Gal Pressman wrote:
> On 28-Feb-19 00:08, Saleem, Shiraz wrote:
> >> Subject: [PATCH rdma-next v2 01/11] RDMA: Add EFA related definitions
> >>
> >> Add EFA node, transport and protocol types to core code.
> >> In addition, EFA driver ID is added to the IOCTL interface uapi.
> >>
> >> Signed-off-by: Gal Pressman <galpress@amazon.com>
> >> drivers/infiniband/core/sysfs.c          | 1 +
> >> drivers/infiniband/core/verbs.c          | 2 ++
> >> include/rdma/ib_verbs.h                  | 8 ++++++--
> >> include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
> >> 4 files changed, 10 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index
> >> 9b6a065bdfa5..f7fac4c81559 100644
> >> +++ b/drivers/infiniband/core/sysfs.c
> >> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
> >> 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev-
> >>> node_type);
> >> 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev-
> >>> node_type);
> >> 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n",
> >> dev->node_type);
> >> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev-
> >>> node_type);
> >> 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev-
> >>> node_type);
> >> 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev-
> >>> node_type);
> >> 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
> >> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index
> >> 5a5e83f5f0fc..fc6f748a1933 100644
> >> +++ b/drivers/infiniband/core/verbs.c
> >> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type
> >> node_type)
> >> 		return RDMA_TRANSPORT_USNIC_UDP;
> >> 	if (node_type == RDMA_NODE_RNIC)
> >> 		return RDMA_TRANSPORT_IWARP;
> >> +	if (node_type == RDMA_NODE_EFA)
> >> +		return RDMA_TRANSPORT_EFA;
> >>
> >> 	return RDMA_TRANSPORT_IB;
> >> }
> >> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
> >> 225cb76d469f..55534aba22bf 100644
> >> +++ b/include/rdma/ib_verbs.h
> >> @@ -108,6 +108,7 @@ enum rdma_node_type {
> >> 	RDMA_NODE_RNIC,
> >> 	RDMA_NODE_USNIC,
> >> 	RDMA_NODE_USNIC_UDP,
> >> +	RDMA_NODE_EFA,
> >> };
> >>
> >> enum {
> >> @@ -119,14 +120,16 @@ enum rdma_transport_type {
> >> 	RDMA_TRANSPORT_IB,
> >> 	RDMA_TRANSPORT_IWARP,
> >> 	RDMA_TRANSPORT_USNIC,
> >> -	RDMA_TRANSPORT_USNIC_UDP
> >> +	RDMA_TRANSPORT_USNIC_UDP,
> >> +	RDMA_TRANSPORT_EFA,
> >> };
> >>
> >> enum rdma_protocol_type {
> >> 	RDMA_PROTOCOL_IB,
> >> 	RDMA_PROTOCOL_IBOE,
> >> 	RDMA_PROTOCOL_IWARP,
> >> -	RDMA_PROTOCOL_USNIC_UDP
> >> +	RDMA_PROTOCOL_USNIC_UDP,
> >> +	RDMA_PROTOCOL_EFA,
> >> };
> >>
> > 
> > Link layer is Ethernet? And layer 3, network type?
> 
> Our link layer is Ethernet, but the wire protocol is not the same as RoCE so it
> was decided to use unspecified link layer.
> The wire protocol is tightly coupled to the wire protocol for EC2’s VPC software
> defined network (EFA is not designed to be used independently of EC2’s VPC data
> plane).

It is possible this should just *_UNSPECIFIED/OTHER or something
general so it can be shared. Neither USNIC nor EFA need these values
for any purpose, AFAIK.

They come out of sysfs and a few APIs they don't use anyhow.

Jason
Gal Pressman March 5, 2019, 3:28 p.m. UTC | #10
On 05-Mar-19 03:32, Jason Gunthorpe wrote:
> On Thu, Feb 28, 2019 at 09:31:07AM +0200, Gal Pressman wrote:
>> On 28-Feb-19 00:08, Saleem, Shiraz wrote:
>>>> Subject: [PATCH rdma-next v2 01/11] RDMA: Add EFA related definitions
>>>>
>>>> Add EFA node, transport and protocol types to core code.
>>>> In addition, EFA driver ID is added to the IOCTL interface uapi.
>>>>
>>>> Signed-off-by: Gal Pressman <galpress@amazon.com>
>>>> drivers/infiniband/core/sysfs.c          | 1 +
>>>> drivers/infiniband/core/verbs.c          | 2 ++
>>>> include/rdma/ib_verbs.h                  | 8 ++++++--
>>>> include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
>>>> 4 files changed, 10 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index
>>>> 9b6a065bdfa5..f7fac4c81559 100644
>>>> +++ b/drivers/infiniband/core/sysfs.c
>>>> @@ -1194,6 +1194,7 @@ static ssize_t node_type_show(struct device *device,
>>>> 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev-
>>>>> node_type);
>>>> 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev-
>>>>> node_type);
>>>> 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n",
>>>> dev->node_type);
>>>> +	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev-
>>>>> node_type);
>>>> 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev-
>>>>> node_type);
>>>> 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev-
>>>>> node_type);
>>>> 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
>>>> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index
>>>> 5a5e83f5f0fc..fc6f748a1933 100644
>>>> +++ b/drivers/infiniband/core/verbs.c
>>>> @@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type
>>>> node_type)
>>>> 		return RDMA_TRANSPORT_USNIC_UDP;
>>>> 	if (node_type == RDMA_NODE_RNIC)
>>>> 		return RDMA_TRANSPORT_IWARP;
>>>> +	if (node_type == RDMA_NODE_EFA)
>>>> +		return RDMA_TRANSPORT_EFA;
>>>>
>>>> 	return RDMA_TRANSPORT_IB;
>>>> }
>>>> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
>>>> 225cb76d469f..55534aba22bf 100644
>>>> +++ b/include/rdma/ib_verbs.h
>>>> @@ -108,6 +108,7 @@ enum rdma_node_type {
>>>> 	RDMA_NODE_RNIC,
>>>> 	RDMA_NODE_USNIC,
>>>> 	RDMA_NODE_USNIC_UDP,
>>>> +	RDMA_NODE_EFA,
>>>> };
>>>>
>>>> enum {
>>>> @@ -119,14 +120,16 @@ enum rdma_transport_type {
>>>> 	RDMA_TRANSPORT_IB,
>>>> 	RDMA_TRANSPORT_IWARP,
>>>> 	RDMA_TRANSPORT_USNIC,
>>>> -	RDMA_TRANSPORT_USNIC_UDP
>>>> +	RDMA_TRANSPORT_USNIC_UDP,
>>>> +	RDMA_TRANSPORT_EFA,
>>>> };
>>>>
>>>> enum rdma_protocol_type {
>>>> 	RDMA_PROTOCOL_IB,
>>>> 	RDMA_PROTOCOL_IBOE,
>>>> 	RDMA_PROTOCOL_IWARP,
>>>> -	RDMA_PROTOCOL_USNIC_UDP
>>>> +	RDMA_PROTOCOL_USNIC_UDP,
>>>> +	RDMA_PROTOCOL_EFA,
>>>> };
>>>>
>>>
>>> Link layer is Ethernet? And layer 3, network type?
>>
>> Our link layer is Ethernet, but the wire protocol is not the same as RoCE so it
>> was decided to use unspecified link layer.
>> The wire protocol is tightly coupled to the wire protocol for EC2’s VPC software
>> defined network (EFA is not designed to be used independently of EC2’s VPC data
>> plane).
> 
> It is possible this should just *_UNSPECIFIED/OTHER or something
> general so it can be shared. Neither USNIC nor EFA need these values
> for any purpose, AFAIK.
> 
> They come out of sysfs and a few APIs they don't use anyhow.

OK, I'll change the EFA stuff to unspecified (to be consistent with the link layer).
I'd rather leave the usnic definitions, but it's up to you, let me know what you
prefer.
Jason Gunthorpe March 5, 2019, 8:16 p.m. UTC | #11
On Tue, Mar 05, 2019 at 05:28:32PM +0200, Gal Pressman wrote:
> >> Our link layer is Ethernet, but the wire protocol is not the same as RoCE so it
> >> was decided to use unspecified link layer.
> >> The wire protocol is tightly coupled to the wire protocol for EC2’s VPC software
> >> defined network (EFA is not designed to be used independently of EC2’s VPC data
> >> plane).
> > 
> > It is possible this should just *_UNSPECIFIED/OTHER or something
> > general so it can be shared. Neither USNIC nor EFA need these values
> > for any purpose, AFAIK.
> > 
> > They come out of sysfs and a few APIs they don't use anyhow.
> 
> OK, I'll change the EFA stuff to unspecified (to be consistent with the link layer).
> I'd rather leave the usnic definitions, but it's up to you, let me know what you
> prefer.

usnic is already abi, we shouldn't change it

Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 9b6a065bdfa5..f7fac4c81559 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1194,6 +1194,7 @@  static ssize_t node_type_show(struct device *device,
 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
+	case RDMA_NODE_EFA:       return sprintf(buf, "%d: EFA\n", dev->node_type);
 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 5a5e83f5f0fc..fc6f748a1933 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -218,6 +218,8 @@  rdma_node_get_transport(enum rdma_node_type node_type)
 		return RDMA_TRANSPORT_USNIC_UDP;
 	if (node_type == RDMA_NODE_RNIC)
 		return RDMA_TRANSPORT_IWARP;
+	if (node_type == RDMA_NODE_EFA)
+		return RDMA_TRANSPORT_EFA;
 
 	return RDMA_TRANSPORT_IB;
 }
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 225cb76d469f..55534aba22bf 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -108,6 +108,7 @@  enum rdma_node_type {
 	RDMA_NODE_RNIC,
 	RDMA_NODE_USNIC,
 	RDMA_NODE_USNIC_UDP,
+	RDMA_NODE_EFA,
 };
 
 enum {
@@ -119,14 +120,16 @@  enum rdma_transport_type {
 	RDMA_TRANSPORT_IB,
 	RDMA_TRANSPORT_IWARP,
 	RDMA_TRANSPORT_USNIC,
-	RDMA_TRANSPORT_USNIC_UDP
+	RDMA_TRANSPORT_USNIC_UDP,
+	RDMA_TRANSPORT_EFA,
 };
 
 enum rdma_protocol_type {
 	RDMA_PROTOCOL_IB,
 	RDMA_PROTOCOL_IBOE,
 	RDMA_PROTOCOL_IWARP,
-	RDMA_PROTOCOL_USNIC_UDP
+	RDMA_PROTOCOL_USNIC_UDP,
+	RDMA_PROTOCOL_EFA,
 };
 
 __attribute_const__ enum rdma_transport_type
@@ -543,6 +546,7 @@  static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
 #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
 #define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
 #define RDMA_CORE_CAP_PROT_USNIC        0x02000000
+#define RDMA_CORE_CAP_PROT_EFA          0x04000000
 
 #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
 					| RDMA_CORE_CAP_PROT_ROCE     \
diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
index 06c34d99be85..26213f49f5c8 100644
--- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
+++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
@@ -102,6 +102,7 @@  enum rdma_driver_id {
 	RDMA_DRIVER_RXE,
 	RDMA_DRIVER_HFI1,
 	RDMA_DRIVER_QIB,
+	RDMA_DRIVER_EFA,
 };
 
 #endif