@@ -54,6 +54,17 @@ config NFP_APP_ABM_NIC
functionality.
Code will be built into the nfp.ko driver.
+config NFP_NET_IPSEC
+ bool "NFP IPsec crypto offload support"
+ depends on NFP
+ depends on XFRM_OFFLOAD
+ default y
+ help
+ Enable driver support IPsec crypto offload on NFP NIC.
+ Say Y, if you are planning to make use of IPsec crypto
+ offload. NOTE that IPsec crypto offload on NFP NIC
+ requires specific FW to work.
+
config NFP_DEBUG
bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers"
depends on NFP
@@ -80,4 +80,6 @@ nfp-objs += \
abm/main.o
endif
+nfp-$(CONFIG_NFP_NET_IPSEC) += crypto/ipsec.o nfd3/ipsec.o
+
nfp-$(CONFIG_NFP_DEBUG) += nfp_net_debugfs.o
@@ -39,4 +39,27 @@ nfp_net_tls_rx_resync_req(struct net_device *netdev,
}
#endif
+/* IPsec related structures and functions */
+struct nfp_ipsec_offload {
+ u32 seq_hi;
+ u32 seq_low;
+ u32 handle;
+};
+
+#ifndef CONFIG_NFP_NET_IPSEC
+static inline void nfp_net_ipsec_init(struct nfp_net *nn)
+{
+}
+
+static inline void nfp_net_ipsec_clean(struct nfp_net *nn)
+{
+}
+#else
+void nfp_net_ipsec_init(struct nfp_net *nn);
+void nfp_net_ipsec_clean(struct nfp_net *nn);
+bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
+ struct nfp_ipsec_offload *offload_info);
+int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb);
+#endif
+
#endif
new file mode 100644
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright (C) 2018 Netronome Systems, Inc */
+/* Copyright (C) 2021 Corigine, Inc */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/netdevice.h>
+#include <asm/unaligned.h>
+#include <linux/ktime.h>
+#include <net/xfrm.h>
+
+#include "../nfp_net_ctrl.h"
+#include "../nfp_net.h"
+#include "crypto.h"
+
+#define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */
+
+static int nfp_net_xfrm_add_state(struct xfrm_state *x)
+{
+ return -EOPNOTSUPP;
+}
+
+static void nfp_net_xfrm_del_state(struct xfrm_state *x)
+{
+}
+
+static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
+{
+ return false;
+}
+
+static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = {
+ .xdo_dev_state_add = nfp_net_xfrm_add_state,
+ .xdo_dev_state_delete = nfp_net_xfrm_del_state,
+ .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok,
+};
+
+void nfp_net_ipsec_init(struct nfp_net *nn)
+{
+ if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
+ return;
+
+ xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC);
+ nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops;
+}
+
+void nfp_net_ipsec_clean(struct nfp_net *nn)
+{
+ if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
+ return;
+ xa_destroy(&nn->xa_ipsec);
+}
+
+bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
+ struct nfp_ipsec_offload *offload_info)
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ struct xfrm_state *x;
+
+ x = xfrm_input_state(skb);
+ if (!x)
+ return false;
+
+ offload_info->seq_hi = xo->seq.hi;
+ offload_info->seq_low = xo->seq.low;
+ offload_info->handle = x->xso.offload_handle;
+
+ return true;
+}
+
+int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)
+{
+ struct net_device *netdev = skb->dev;
+ struct xfrm_offload *xo;
+ struct xfrm_state *x;
+ struct sec_path *sp;
+ struct nfp_net *nn;
+ u32 saidx;
+
+ nn = netdev_priv(netdev);
+
+ saidx = meta->ipsec_saidx - 1;
+ if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT)
+ return -EINVAL;
+
+ sp = secpath_set(skb);
+ if (unlikely(!sp))
+ return -ENOMEM;
+
+ xa_lock(&nn->xa_ipsec);
+ x = xa_load(&nn->xa_ipsec, saidx);
+ xa_unlock(&nn->xa_ipsec);
+ if (!x)
+ return -EINVAL;
+
+ xfrm_state_hold(x);
+ sp->xvec[sp->len++] = x;
+ sp->olen++;
+ xo = xfrm_offload(skb);
+ xo->flags = CRYPTO_DONE;
+ xo->status = CRYPTO_SUCCESS;
+
+ return 0;
+}
@@ -4,6 +4,7 @@
#include <linux/bpf_trace.h>
#include <linux/netdevice.h>
#include <linux/bitfield.h>
+#include <net/xfrm.h>
#include "../nfp_app.h"
#include "../nfp_net.h"
@@ -167,28 +168,34 @@ nfp_nfd3_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
u64_stats_update_end(&r_vec->tx_sync);
}
-static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64 tls_handle)
+static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb,
+ u64 tls_handle, bool *ipsec)
{
struct metadata_dst *md_dst = skb_metadata_dst(skb);
+ struct nfp_ipsec_offload offload_info;
unsigned char *data;
bool vlan_insert;
u32 meta_id = 0;
int md_bytes;
- if (unlikely(md_dst || tls_handle)) {
- if (unlikely(md_dst && md_dst->type != METADATA_HW_PORT_MUX))
- md_dst = NULL;
- }
+#ifdef CONFIG_NFP_NET_IPSEC
+ if (xfrm_offload(skb))
+ *ipsec = nfp_net_ipsec_tx_prep(dp, skb, &offload_info);
+#endif
+
+ if (unlikely(md_dst && md_dst->type != METADATA_HW_PORT_MUX))
+ md_dst = NULL;
vlan_insert = skb_vlan_tag_present(skb) && (dp->ctrl & NFP_NET_CFG_CTRL_TXVLAN_V2);
- if (!(md_dst || tls_handle || vlan_insert))
+ if (!(md_dst || tls_handle || vlan_insert || *ipsec))
return 0;
md_bytes = sizeof(meta_id) +
!!md_dst * NFP_NET_META_PORTID_SIZE +
!!tls_handle * NFP_NET_META_CONN_HANDLE_SIZE +
- vlan_insert * NFP_NET_META_VLAN_SIZE;
+ vlan_insert * NFP_NET_META_VLAN_SIZE +
+ *ipsec * NFP_NET_META_IPSEC_FIELD_SIZE; /* IPsec has 12 bytes of metadata */
if (unlikely(skb_cow_head(skb, md_bytes)))
return -ENOMEM;
@@ -218,6 +225,19 @@ static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64
meta_id <<= NFP_NET_META_FIELD_SIZE;
meta_id |= NFP_NET_META_VLAN;
}
+ if (*ipsec) {
+ /* IPsec has three consecutive 4-bit IPsec metadata types,
+ * so in total IPsec has three 4 bytes of metadata.
+ */
+ data -= NFP_NET_META_IPSEC_SIZE;
+ put_unaligned_be32(offload_info.seq_hi, data);
+ data -= NFP_NET_META_IPSEC_SIZE;
+ put_unaligned_be32(offload_info.seq_low, data);
+ data -= NFP_NET_META_IPSEC_SIZE;
+ put_unaligned_be32(offload_info.handle - 1, data);
+ meta_id <<= NFP_NET_META_IPSEC_FIELD_SIZE;
+ meta_id |= NFP_NET_META_IPSEC << 8 | NFP_NET_META_IPSEC << 4 | NFP_NET_META_IPSEC;
+ }
data -= sizeof(meta_id);
put_unaligned_be32(meta_id, data);
@@ -246,6 +266,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
dma_addr_t dma_addr;
unsigned int fsize;
u64 tls_handle = 0;
+ bool ipsec = false;
u16 qidx;
dp = &nn->dp;
@@ -273,7 +294,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
return NETDEV_TX_OK;
}
- md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle);
+ md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle, &ipsec);
if (unlikely(md_bytes < 0))
goto err_flush;
@@ -312,6 +333,8 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
}
+ if (ipsec)
+ nfp_nfd3_ipsec_tx(txd, skb);
/* Gather DMA */
if (nr_frags > 0) {
__le64 second_half;
@@ -764,6 +787,15 @@ nfp_nfd3_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,
return false;
data += sizeof(struct nfp_net_tls_resync_req);
break;
+#ifdef CONFIG_NFP_NET_IPSEC
+ case NFP_NET_META_IPSEC:
+ /* Note: IPsec packet will have zero saidx, so need add 1
+ * to indicate packet is IPsec packet within driver.
+ */
+ meta->ipsec_saidx = get_unaligned_be32(data) + 1;
+ data += 4;
+ break;
+#endif
default:
return true;
}
@@ -876,12 +908,11 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget)
struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
struct nfp_net_tx_ring *tx_ring;
struct bpf_prog *xdp_prog;
+ int idx, pkts_polled = 0;
bool xdp_tx_cmpl = false;
unsigned int true_bufsz;
struct sk_buff *skb;
- int pkts_polled = 0;
struct xdp_buff xdp;
- int idx;
xdp_prog = READ_ONCE(dp->xdp_prog);
true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz;
@@ -1081,6 +1112,13 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget)
continue;
}
+#ifdef CONFIG_NFP_NET_IPSEC
+ if (meta.ipsec_saidx != 0 && unlikely(nfp_net_ipsec_rx(&meta, skb))) {
+ nfp_nfd3_rx_drop(dp, r_vec, rx_ring, NULL, skb);
+ continue;
+ }
+#endif
+
if (meta_len_xdp)
skb_metadata_set(skb, meta_len_xdp);
new file mode 100644
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright (C) 2018 Netronome Systems, Inc */
+/* Copyright (C) 2021 Corigine, Inc */
+
+#include <net/xfrm.h>
+
+#include "../nfp_net.h"
+#include "nfd3.h"
+
+void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb)
+{
+ struct xfrm_state *x = xfrm_input_state(skb);
+
+ if (x->xso.dev && (x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)) {
+ txd->flags |= NFD3_DESC_TX_CSUM | NFD3_DESC_TX_IP4_CSUM |
+ NFD3_DESC_TX_TCP_CSUM | NFD3_DESC_TX_UDP_CSUM;
+ }
+}
@@ -103,4 +103,12 @@ void nfp_nfd3_rx_ring_fill_freelist(struct nfp_net_dp *dp,
void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf);
int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget);
+#ifndef CONFIG_NFP_NET_IPSEC
+static inline void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb)
+{
+}
+#else
+void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb);
+#endif
+
#endif
@@ -263,6 +263,10 @@ struct nfp_meta_parsed {
u8 tpid;
u16 tci;
} vlan;
+
+#ifdef CONFIG_NFP_NET_IPSEC
+ u32 ipsec_saidx;
+#endif
};
struct nfp_net_rx_hash {
@@ -584,6 +588,7 @@ struct nfp_net_dp {
* @qcp_cfg: Pointer to QCP queue used for configuration notification
* @tx_bar: Pointer to mapped TX queues
* @rx_bar: Pointer to mapped FL/RX queues
+ * @xa_ipsec: IPsec xarray SA data
* @tlv_caps: Parsed TLV capabilities
* @ktls_tx_conn_cnt: Number of offloaded kTLS TX connections
* @ktls_rx_conn_cnt: Number of offloaded kTLS RX connections
@@ -672,6 +677,10 @@ struct nfp_net {
u8 __iomem *tx_bar;
u8 __iomem *rx_bar;
+#ifdef CONFIG_NFP_NET_IPSEC
+ struct xarray xa_ipsec;
+#endif
+
struct nfp_net_tlv_caps tlv_caps;
unsigned int ktls_tx_conn_cnt;
@@ -2566,6 +2566,8 @@ int nfp_net_init(struct nfp_net *nn)
err = nfp_net_tls_init(nn);
if (err)
goto err_clean_mbox;
+
+ nfp_net_ipsec_init(nn);
}
nfp_net_vecs_init(nn);
@@ -2589,6 +2591,7 @@ void nfp_net_clean(struct nfp_net *nn)
return;
unregister_netdev(nn->dp.netdev);
+ nfp_net_ipsec_clean(nn);
nfp_ccm_mbox_clean(nn);
nfp_net_reconfig_wait_posted(nn);
}
@@ -48,6 +48,7 @@
#define NFP_NET_META_CSUM 6 /* checksum complete type */
#define NFP_NET_META_CONN_HANDLE 7
#define NFP_NET_META_RESYNC_INFO 8 /* RX resync info request */
+#define NFP_NET_META_IPSEC 9 /* IPsec SA index for tx and rx */
#define NFP_META_PORT_ID_CTRL ~0U
@@ -55,6 +56,8 @@
#define NFP_NET_META_VLAN_SIZE 4
#define NFP_NET_META_PORTID_SIZE 4
#define NFP_NET_META_CONN_HANDLE_SIZE 8
+#define NFP_NET_META_IPSEC_SIZE 4
+#define NFP_NET_META_IPSEC_FIELD_SIZE 12
/* Hash type pre-pended when a RSS hash was computed */
#define NFP_NET_RSS_NONE 0
#define NFP_NET_RSS_IPV4 1
@@ -263,6 +266,7 @@
*/
#define NFP_NET_CFG_CTRL_WORD1 0x0098
#define NFP_NET_CFG_CTRL_PKT_TYPE (0x1 << 0) /* Pkttype offload */
+#define NFP_NET_CFG_CTRL_IPSEC (0x1 << 1) /* IPsec offload */
#define NFP_NET_CFG_CAP_WORD1 0x00a4