From patchwork Fri Jan 18 07:14:02 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Atul Gupta X-Patchwork-Id: 10769489 X-Patchwork-Delegate: herbert@gondor.apana.org.au Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 469096C5 for ; Fri, 18 Jan 2019 07:14:11 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2940528742 for ; Fri, 18 Jan 2019 07:14:11 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 1015328FEA; Fri, 18 Jan 2019 07:14:11 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 4022428742 for ; Fri, 18 Jan 2019 07:14:09 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727194AbfARHOI (ORCPT ); Fri, 18 Jan 2019 02:14:08 -0500 Received: from stargate.chelsio.com ([12.32.117.8]:7414 "EHLO stargate.chelsio.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727170AbfARHOI (ORCPT ); Fri, 18 Jan 2019 02:14:08 -0500 Received: from beagle7.asicdesigners.com (beagle7.asicdesigners.com [10.192.192.157]) by stargate.chelsio.com (8.13.8/8.13.8) with ESMTP id x0I7E3Yc006368; Thu, 17 Jan 2019 23:14:03 -0800 From: Atul Gupta To: davem@davemloft.net, herbert@gondor.apana.org.au, linux-crypto@vger.kernel.org, netdev@vger.kernel.org, dt@chelsio.com Cc: atul.gupta@chelsio.com Subject: [RFC patch 2/4] crypto/chelsio/chtls: hardware connect API Date: Thu, 17 Jan 2019 23:14:02 -0800 Message-Id: <20190118071402.9587-1-atul.gupta@chelsio.com> X-Mailer: git-send-email 2.20.0.rc2.7.g965798d MIME-Version: 1.0 Sender: linux-crypto-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Hardware specific implementation for TLS client processing. Added connect routine to prepare hardware for TLS client handshake. Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/chtls/chtls.h | 6 +- drivers/crypto/chelsio/chtls/chtls_cm.c | 533 ++++++++++++++++++++++++-- drivers/crypto/chelsio/chtls/chtls_cm.h | 6 +- drivers/crypto/chelsio/chtls/chtls_hw.c | 6 +- drivers/crypto/chelsio/chtls/chtls_main.c | 161 ++++++++ drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 2 + net/core/secure_seq.c | 1 + 7 files changed, 684 insertions(+), 31 deletions(-) diff --git a/drivers/crypto/chelsio/chtls/chtls.h b/drivers/crypto/chelsio/chtls/chtls.h index 59bb67d..9742613 100644 --- a/drivers/crypto/chelsio/chtls/chtls.h +++ b/drivers/crypto/chelsio/chtls/chtls.h @@ -136,6 +136,8 @@ struct chtls_dev { struct idr stid_idr; spinlock_t idr_lock ____cacheline_aligned_in_smp; + spinlock_t aidr_lock ____cacheline_aligned_in_smp; + struct idr aidr; /* ATID id space */ struct net_device *egr_dev[NCHAN * 2]; struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS]; @@ -191,6 +193,7 @@ struct chtls_sock { struct net_device *egress_dev; /* TX_CHAN for act open retry */ struct sk_buff_head txq; + struct sk_buff_head ooq; struct sk_buff *wr_skb_head; struct sk_buff *wr_skb_tail; struct sk_buff *ctrl_skb_cache; @@ -206,6 +209,7 @@ struct chtls_sock { u32 txq_idx; u32 rss_qid; u32 tid; + u32 neg_adv_tid; u32 idr; u32 mss; u32 ulp_mode; @@ -389,7 +393,7 @@ static inline bool csk_conn_inline(const struct chtls_sock *csk) static inline int csk_flag(const struct sock *sk, enum csk_flags flag) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk = sk->sk_user_data; if (!csk_conn_inline(csk)) return 0; diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c index 2e11b0d..b11c991 100644 --- a/drivers/crypto/chelsio/chtls/chtls_cm.c +++ b/drivers/crypto/chelsio/chtls/chtls_cm.c @@ -29,6 +29,8 @@ #include "chtls.h" #include "chtls_cm.h" +static void chtls_connect_req_arp_failure(void *handle, struct sk_buff *skb); + /* * State transitions and actions for close. Note that if we are in SYN_SENT * we remain in that state as we cannot control a connection while it's in @@ -66,6 +68,7 @@ static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev) kref_init(&csk->kref); csk->cdev = cdev; skb_queue_head_init(&csk->txq); + skb_queue_head_init(&csk->ooq); csk->wr_skb_head = NULL; csk->wr_skb_tail = NULL; csk->mss = MAX_MSS; @@ -85,6 +88,60 @@ static void chtls_sock_release(struct kref *ref) kfree(csk); } +static int bh_insert_handle(struct chtls_dev *cdev, struct sock *sk, + int tid) +{ + int id; + + spin_lock_bh(&cdev->idr_lock); + id = idr_alloc(&cdev->hwtid_idr, sk, tid, tid + 1, GFP_NOWAIT); + spin_unlock_bh(&cdev->idr_lock); + return id; +} + +static int sk_insert_tid(struct chtls_dev *cdev, struct sock *sk, + unsigned int tid) +{ + int id; + + sock_hold(sk); + cxgb4_insert_tid(cdev->tids, sk, tid, sk->sk_family); + id = bh_insert_handle(cdev, sk, tid); + return id; +} + +#define __FIXUP_WR_MIT_CPL(_w, cpl, _tid) do { \ + typeof(_w) (w) = (_w); \ + typeof(_tid) (tid) = (_tid); \ + (w)->wr.wr_mid = \ + htonl(FW_WR_LEN16_V(FW_WR_LEN16_G(ntohl((w)->wr.wr_mid))) | \ + FW_WR_FLOWID_V(tid)); \ + OPCODE_TID(w) = htonl(MK_OPCODE_TID(cpl, tid)); \ +} while (0) + +#define __FIXUP_FLOWC_WR(_flowc, tid) do { \ + typeof(_flowc) (flowc) = (_flowc); \ + (flowc)->flowid_len16 = \ + htonl(FW_WR_LEN16_V(FW_WR_LEN16_G(ntohl((flowc)->flowid_len16))) | \ + FW_WR_FLOWID_V(tid)); \ +} while (0) + +static void fixup_and_send_ofo(struct chtls_sock *csk, unsigned int tid) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&csk->ooq)) != NULL) { + struct fw_flowc_wr *flowc = cplhdr(skb); + struct cpl_close_con_req *p = cplhdr(skb); + + if (FW_WR_OP_G(ntohl(flowc->op_to_nparams)) == FW_FLOWC_WR) + __FIXUP_FLOWC_WR(flowc, tid); + else + __FIXUP_WR_MIT_CPL(p, p->ot.opcode, tid); + cxgb4_ofld_send(csk->egress_dev, skb); + } +} + static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev, struct sock *sk) { @@ -108,7 +165,7 @@ static void assign_rxopt(struct sock *sk, unsigned int opt) struct chtls_sock *csk; struct tcp_sock *tp; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tp = tcp_sk(sk); cdev = csk->cdev; @@ -142,9 +199,10 @@ static void chtls_purge_receive_queue(struct sock *sk) static void chtls_purge_write_queue(struct sock *sk) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk; struct sk_buff *skb; + csk = sk->sk_user_data; while ((skb = __skb_dequeue(&csk->txq))) { sk->sk_wmem_queued -= skb->truesize; __kfree_skb(skb); @@ -153,10 +211,12 @@ static void chtls_purge_write_queue(struct sock *sk) static void chtls_purge_recv_queue(struct sock *sk) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_hws *tlsk = &csk->tlshws; + struct chtls_sock *csk; + struct chtls_hws *tlsk; struct sk_buff *skb; + csk = sk->sk_user_data; + tlsk = &csk->tlshws; while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) { skb_dst_set(skb, NULL); kfree_skb(skb); @@ -208,8 +268,9 @@ static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb) static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk; + csk = sk->sk_user_data; if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) || !csk->cdev)) { if (sk->sk_state == TCP_SYN_RECV) @@ -264,7 +325,7 @@ static void chtls_close_conn(struct sock *sk) unsigned int len; len = roundup(sizeof(struct cpl_close_con_req), 16); - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tid = csk->tid; skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); @@ -302,8 +363,7 @@ void chtls_close(struct sock *sk, long timeout) int data_lost, prev_state; struct chtls_sock *csk; - csk = rcu_dereference_sk_user_data(sk); - + csk = sk->sk_user_data; lock_sock(sk); sk->sk_shutdown |= SHUTDOWN_MASK; @@ -442,7 +502,7 @@ void chtls_destroy_sock(struct sock *sk) { struct chtls_sock *csk; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; chtls_purge_recv_queue(sk); csk->ulp_mode = ULP_MODE_NONE; chtls_purge_write_queue(sk); @@ -705,6 +765,22 @@ static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb) return 0; } +static void conn_remove_handle(struct chtls_dev *cdev, int tid) +{ + spin_lock(&cdev->aidr_lock); + idr_remove(&cdev->aidr, tid); + spin_unlock(&cdev->aidr_lock); +} + +static void free_atid(struct chtls_sock *csk, struct chtls_dev *cdev, + unsigned int atid) +{ + conn_remove_handle(cdev, atid); + cxgb4_free_atid(cdev->tids, atid); + sock_put(csk->sk); + kref_put(&csk->kref, chtls_sock_release); +} + static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) { struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR; @@ -732,7 +808,7 @@ static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) static void chtls_release_resources(struct sock *sk) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk = sk->sk_user_data; struct chtls_dev *cdev = csk->cdev; unsigned int tid = csk->tid; struct tid_info *tids; @@ -905,6 +981,304 @@ static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp) return wscale; } +/* Active Open Processing */ +static int chtls_conn_insert_hdl(struct chtls_dev *cdev, struct sock *sk, + int tid) +{ + int id; + + idr_preload(GFP_KERNEL); + spin_lock_bh(&cdev->aidr_lock); + id = idr_alloc(&cdev->aidr, sk, tid, tid + 1, GFP_NOWAIT); + spin_unlock_bh(&cdev->aidr_lock); + idr_preload_end(); + return id; +} + +static void chtls_act_open_fail(struct sock *sk, int errno) +{ + sk->sk_err = errno; + sk->sk_error_report(sk); + chtls_release_resources(sk); + chtls_conn_done(sk); + TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); +} + +static void chtls_deferred_connect(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct sock *sk = skb->sk; + struct inet_sock *inet = inet_sk(sk); + struct tcp_sock *tp = tcp_sk(sk); + int err; + + kfree_skb(skb); + lock_sock(sk); + if (sk->sk_state == TCP_SYN_SENT) { + if (sk->sk_user_data) + chtls_release_resources(sk); + if (!tp->write_seq) { + if (sk->sk_family == AF_INET) + tp->write_seq = (prandom_u32() & ~7UL) - 1; + } + inet->inet_id = tp->write_seq ^ jiffies; + err = tcp_connect(sk); + if (err) + goto failure; + } + release_sock(sk); + return; +failure: + tcp_set_state(sk, TCP_CLOSE); + sk->sk_route_caps = 0; + inet->inet_dport = 0; + sk->sk_err = err; + sk->sk_error_report(sk); + TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); + release_sock(sk); +} + +static int act_open_rpl_status_to_errno(int status) +{ + switch (status) { + case CPL_ERR_CONN_RESET: + return -ECONNREFUSED; + case CPL_ERR_ARP_MISS: + return -EHOSTUNREACH; + case CPL_ERR_CONN_TIMEDOUT: + return -ETIMEDOUT; + case CPL_ERR_TCAM_FULL: + return -ENOMEM; + case CPL_ERR_CONN_EXIST: + return -EADDRINUSE; + default: + return -EIO; + } +} + +static unsigned long long calc_opt0(struct sock *sk, int nagle) +{ + const struct tcp_sock *tp; + struct chtls_sock *csk; + + csk = sk->sk_user_data; + tp = tcp_sk(sk); + + if (likely(nagle == -1)) + nagle = ((tp->nonagle & TCP_NAGLE_OFF) == 0); + + return NAGLE_V(nagle) | + TCAM_BYPASS_F | + KEEP_ALIVE_V(sock_flag(sk, SOCK_KEEPOPEN) != 0) | + WND_SCALE_V(RCV_WSCALE(tp)) | + MSS_IDX_V(csk->mtu_idx) | + DSCP_V((inet_sk(sk)->tos >> 2) & 0x3F) | + ULP_MODE_V(ULP_MODE_TLS) | + RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M)); +} + +static void chtls_act_open_rqst(struct sock *sk, struct sk_buff *skb, + unsigned int qid_atid, + const struct l2t_entry *e) +{ + struct cpl_t6_act_open_req *req; + struct chtls_sock *csk; + unsigned int opt2; + u32 isn; + + csk = sk->sk_user_data; + req = (struct cpl_t6_act_open_req *)__skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); + set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); + req->local_port = inet_sk(sk)->inet_sport; + req->peer_port = inet_sk(sk)->inet_dport; + req->local_ip = inet_sk(sk)->inet_saddr; + req->peer_ip = inet_sk(sk)->inet_daddr; + req->opt0 = cpu_to_be64(calc_opt0(sk, 0) | + L2T_IDX_V(e->idx) | + SMAC_SEL_V(csk->smac_idx) | + ULP_MODE_V(csk->ulp_mode) | + TX_CHAN_V(csk->tx_chan)); + isn = (prandom_u32() & ~7UL) - 1; + req->rsvd = cpu_to_be32(isn); + req->params = + cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(csk->egress_dev, + csk->l2t_entry))); + opt2 = RX_CHANNEL_V(0) | + TX_QUEUE_V(csk->cdev->lldi->tx_modq[csk->tx_chan]) | + RSS_QUEUE_VALID_F | + RSS_QUEUE_V(csk->rss_qid) | + T5_ISS_F | + RX_FC_DISABLE_F | + T5_OPT_2_VALID_F | + RX_FC_VALID_F; + + if (sock_net(sk)->ipv4.sysctl_tcp_window_scaling) + opt2 |= WND_SCALE_EN_F; + if (sock_net(sk)->ipv4.sysctl_tcp_timestamps) + opt2 |= TSTAMPS_EN_F; + if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK) + opt2 |= CCTRL_ECN_F; + if (sock_net(sk)->ipv4.sysctl_tcp_sack) + opt2 |= SACK_EN_F; + opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); + req->opt2 = cpu_to_be32(opt2); + req->rsvd2 = cpu_to_be32(0); + req->opt3 = cpu_to_be32(0); +} + +static void act_open_retry_timer(struct timer_list *t) +{ + struct inet_connection_sock *icsk; + struct sk_buff *skb; + struct sock *sk; + int len; + + sk = from_timer(sk, t, sk_timer); + icsk = inet_csk(sk); + bh_lock_sock(sk); + if (sock_owned_by_user(sk)) { + sk_reset_timer(sk, &icsk->icsk_retransmit_timer, + jiffies + HZ / 20); + } else { + len = roundup(sizeof(struct cpl_t6_act_open_req6), 16); + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) { + chtls_act_open_fail(sk, ENOMEM); + } else { + struct chtls_sock *csk; + struct chtls_dev *cdev; + unsigned int qid_atid; + + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + qid_atid = csk->rss_qid << 14 | csk->tid; + skb->sk = sk; + t4_set_arp_err_handler(skb, NULL, + chtls_connect_req_arp_failure); + chtls_act_open_rqst(sk, skb, qid_atid, csk->l2t_entry); + cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); + } + } + bh_unlock_sock(sk); + sock_put(sk); +} + +/* + * Add an skb to the deferred skb queue for processing from process context. + */ +static void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev, + defer_handler_t handler) +{ + DEFERRED_SKB_CB(skb)->handler = handler; + spin_lock_bh(&cdev->deferq.lock); + __skb_queue_tail(&cdev->deferq, skb); + if (skb_queue_len(&cdev->deferq) == 1) + schedule_work(&cdev->deferq_task); + spin_unlock_bh(&cdev->deferq.lock); +} + +static void chtls_active_open_rpl(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_act_open_rpl *rpl = cplhdr(skb) + RSS_HDR; + struct inet_connection_sock *icsk; + struct chtls_dev *cdev; + struct chtls_sock *csk; + unsigned int status; + int err; + + icsk = inet_csk(sk); + status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status)); + if (is_neg_adv(status)) { + struct chtls_dev *cdev; + unsigned int tid; + + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + tid = GET_TID(rpl); + + if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) { + if (!lookup_tid(cdev->tids, tid)) + csk->idr = sk_insert_tid(cdev, sk, tid); + } + csk->neg_adv_tid = tid; + fixup_and_send_ofo(csk, tid); + kfree_skb(skb); + return; + } + + if (status) { + if (status == CPL_ERR_CONN_EXIST && + icsk->icsk_retransmit_timer.function != + act_open_retry_timer) { + icsk->icsk_retransmit_timer.function = + act_open_retry_timer; + sk_reset_timer(sk, &icsk->icsk_retransmit_timer, + jiffies + HZ / 2); + kfree_skb(skb); + } else if (status == CPL_ERR_TCAM_PARITY || + status == CPL_ERR_TCAM_FULL) { + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + skb->sk = sk; + chtls_defer_reply(skb, cdev, chtls_deferred_connect); + } else { + err = act_open_rpl_status_to_errno(status); + if (err == EADDRINUSE) { + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + skb->sk = sk; + chtls_defer_reply(skb, cdev, + chtls_deferred_connect); + } + } + } else { + kfree_skb(skb); + } +} + +static void chtls_connect_req_arp_failure(void *handle, struct sk_buff *skb) +{ + struct sock *sk = skb->sk; + + sock_hold(sk); + bh_lock_sock(sk); + if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) { + if (!sock_owned_by_user(sk)) { + chtls_act_open_fail(sk, EHOSTUNREACH); + __kfree_skb(skb); + } else { + struct cpl_act_open_rpl *rpl = cplhdr(skb) + RSS_HDR; + + rpl->ot.opcode = CPL_ACT_OPEN_RPL; + rpl->atid_status = CPL_ERR_ARP_MISS; + BLOG_SKB_CB(skb)->backlog_rcv = chtls_active_open_rpl; + __sk_add_backlog(sk, skb); + } + } + bh_unlock_sock(sk); + sock_put(sk); +} + +static void chtls_write_space(struct sock *sk) +{ + struct socket *sock = sk->sk_socket; + struct socket_wq *wq; + + if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) && sock) { + clear_bit(SOCK_NOSPACE, &sock->flags); + rcu_read_lock(); + wq = rcu_dereference(sk->sk_wq); + if (skwq_has_sleeper(sk->sk_wq)) + wake_up_interruptible_poll(&wq->wait, POLLOUT | + POLLWRNORM | + POLLWRBAND); + if (wq && wq->fasync_list && !(sk->sk_shutdown & SEND_SHUTDOWN)) + sock_wake_async(wq, SOCK_WAKE_SPACE, POLL_OUT); + rcu_read_unlock(); + } +} + static void chtls_pass_accept_rpl(struct sk_buff *skb, struct cpl_pass_accept_req *req, unsigned int tid) @@ -1008,6 +1382,114 @@ static void chtls_set_tcp_window(struct chtls_sock *csk) csk->snd_win *= scale; } +int chtls_active_open(struct chtls_dev *cdev, struct sock *sk, + struct net_device *ndev) +{ + struct dst_entry *dst = __sk_dst_get(sk); + struct tcp_sock *tp = tcp_sk(sk); + struct chtls_sock *csk; + unsigned int qid_atid; + struct sk_buff *skb; + struct neighbour *n; + unsigned int len; + struct net *net; + bool use_ecn; + u16 port_id; + int rxq_idx; + int step; + int atid; + int id; + + csk = chtls_sock_create(cdev); + if (!csk) + return -ENOMEM; + + atid = cxgb4_alloc_atid(cdev->tids, csk); + if (atid < 0) + goto free_csk; + + id = chtls_conn_insert_hdl(cdev, sk, atid); + if (id < 0) + goto free_atid; + + sock_hold(sk); + csk->sk = sk; + csk->egress_dev = ndev; + sk->sk_user_data = csk; + if (sk->sk_family == AF_INET) { + n = dst_neigh_lookup(dst, &inet_sk(sk)->inet_daddr); + if (!n) + goto free_atid; + } + port_id = cxgb4_port_idx(ndev); + + csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0); + if (!csk->l2t_entry) + goto free_atid; + neigh_release(n); + net = sock_net(sk); + tp->ecn_flags = 0; + use_ecn = (net->ipv4.sysctl_tcp_ecn == 1) || tcp_ca_needs_ecn(sk); + if (!use_ecn) { + if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) + use_ecn = true; + } + if (use_ecn) + tp->ecn_flags = TCP_ECN_OK; + + len = roundup(sizeof(struct cpl_t6_act_open_req6), 16); + skb = alloc_skb(len, GFP_KERNEL); + if (!skb) + goto free_atid; + skb->sk = sk; + t4_set_arp_err_handler(skb, sk, chtls_connect_req_arp_failure); + kref_get(&csk->kref); + + chtls_install_cpl_ops(sk); + sk->sk_backlog_rcv = chtls_backlog_rcv; + csk->tx_chan = cxgb4_port_chan(ndev); + csk->tid = atid; + if (!tp->window_clamp) + tp->window_clamp = dst_metric(dst, RTAX_WINDOW); + chtls_write_space(sk); + csk_set_flag(csk, CSK_CONN_INLINE); + csk->wr_max_credits = 64; + csk->wr_credits = 64; + csk->wr_unacked = 0; + csk->delack_mode = 0; + chtls_set_tcp_window(csk); + tp->rcv_wnd = csk->rcv_win; + csk->sndbuf = csk->snd_win; + csk->ulp_mode = ULP_MODE_TLS; + step = cdev->lldi->nrxq / cdev->lldi->nchan; + csk->port_id = port_id; + csk->rss_qid = cdev->lldi->rxq_ids[port_id * step]; + rxq_idx = port_id * step; + csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx : + port_id * step; + csk->mtu_idx = chtls_select_mss(csk, dst_mtu(dst), 0); + RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(sk), + sock_net(sk)-> + ipv4.sysctl_tcp_window_scaling, + tp->window_clamp); + sk->sk_err = 0; + sock_reset_flag(sk, SOCK_DONE); + TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS); + csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; + qid_atid = csk->rss_qid << 14; + qid_atid |= (unsigned int)atid; + + chtls_act_open_rqst(sk, skb, qid_atid, csk->l2t_entry); + cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); + return 0; +free_atid: + free_atid(csk, cdev, atid); +free_csk: + chtls_sock_release(&csk->kref); + + return -1; +} + static struct sock *chtls_recv_sock(struct sock *lsk, struct request_sock *oreq, void *network_hdr, @@ -1514,7 +1996,7 @@ static void chtls_recv_data(struct sock *sk, struct sk_buff *skb) struct chtls_sock *csk; struct tcp_sock *tp; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tp = tcp_sk(sk); if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { @@ -1577,7 +2059,7 @@ static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb) struct chtls_hws *tlsk; struct tcp_sock *tp; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tlsk = &csk->tlshws; tp = tcp_sk(sk); @@ -1640,7 +2122,7 @@ static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb) struct tcp_sock *tp; cmp_cpl = cplhdr(skb); - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tlsk = &csk->tlshws; tp = tcp_sk(sk); @@ -1704,7 +2186,7 @@ static void chtls_timewait(struct sock *sk) static void chtls_peer_close(struct sock *sk, struct sk_buff *skb) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk = sk->sk_user_data; sk->sk_shutdown |= RCV_SHUTDOWN; sock_set_flag(sk, SOCK_DONE); @@ -1746,7 +2228,7 @@ static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb) struct chtls_sock *csk; struct tcp_sock *tp; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tp = tcp_sk(sk); tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ @@ -1825,7 +2307,7 @@ static void send_abort_rpl(struct sock *sk, struct sk_buff *skb, struct sk_buff *reply_skb; struct chtls_sock *csk; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl), GFP_KERNEL); @@ -1874,7 +2356,7 @@ static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb, struct chtls_sock *csk; unsigned int tid; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; tid = GET_TID(req); reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any()); @@ -1909,7 +2391,7 @@ static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb) int queue; child = skb->sk; - csk = rcu_dereference_sk_user_data(child); + csk = sk->sk_user_data; queue = csk->txq_idx; skb->sk = NULL; @@ -2006,7 +2488,7 @@ static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb) struct chtls_sock *csk; struct chtls_dev *cdev; - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; cdev = csk->cdev; if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { @@ -2067,9 +2549,11 @@ static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb) static struct sk_buff *dequeue_wr(struct sock *sk) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb = csk->wr_skb_head; + struct chtls_sock *csk; + struct sk_buff *skb; + csk = sk->sk_user_data; + skb = csk->wr_skb_head; if (likely(skb)) { /* Don't bother clearing the tail */ csk->wr_skb_head = WR_SKB_CB(skb)->next_wr; @@ -2105,10 +2589,11 @@ static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb) if (unlikely(credits < csum)) { pskb->csum = (__force __wsum)(csum - credits); break; + } else { + dequeue_wr(sk); + credits -= csum; + kfree_skb(pskb); } - dequeue_wr(sk); - credits -= csum; - kfree_skb(pskb); } if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { if (unlikely(before(snd_una, tp->snd_una))) { diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.h b/drivers/crypto/chelsio/chtls/chtls_cm.h index 78eb3af..ca3ccb7 100644 --- a/drivers/crypto/chelsio/chtls/chtls_cm.h +++ b/drivers/crypto/chelsio/chtls/chtls_cm.h @@ -106,9 +106,6 @@ struct deferred_skb_cb { #define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld) #define skb_ulp_tls_iv_imm(skb) (ULP_SKB_CB(skb)->ulp.tls.iv) -void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev, - defer_handler_t handler); - /* * Returns true if the socket is in one of the supplied states. */ @@ -200,4 +197,7 @@ static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb) WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb; csk->wr_skb_tail = skb; } + +int chtls_active_open(struct chtls_dev *cdev, struct sock *sk, + struct net_device *ndev); #endif diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c index 4909607..6266b9e 100644 --- a/drivers/crypto/chelsio/chtls/chtls_hw.c +++ b/drivers/crypto/chelsio/chtls/chtls_hw.c @@ -50,7 +50,7 @@ static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word, unsigned int wrlen; wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen); __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply); @@ -78,7 +78,7 @@ static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val) return -ENOMEM; credits_needed = DIV_ROUND_UP(wrlen, 16); - csk = rcu_dereference_sk_user_data(sk); + csk = sk->sk_user_data; __set_tcb_field(sk, skb, word, mask, val, 0, 1); skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); @@ -166,7 +166,7 @@ static int get_new_keyid(struct chtls_sock *csk, u32 optname) void free_tls_keyid(struct sock *sk) { - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_sock *csk = sk->sk_user_data; struct net_device *dev = csk->egress_dev; struct chtls_dev *cdev = csk->cdev; struct chtls_hws *hws; diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c index 563f8fe..9328bde 100644 --- a/drivers/crypto/chelsio/chtls/chtls_main.c +++ b/drivers/crypto/chelsio/chtls/chtls_main.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -161,6 +162,163 @@ static void chtls_destroy_hash(struct tls_device *dev, struct sock *sk) chtls_stop_listen(cdev, sk); } +static int chtls_ndev_found(struct chtls_dev *cdev, struct net_device *ndev) +{ + int i; + + for (i = 0; i < cdev->lldi->nports; i++) + if (ndev == cdev->ports[i]) + return 1; + return 0; +} + +static int chtls_connect(struct tls_device *dev, struct sock *sk, + struct sockaddr *uaddr, int addr_len) +{ + struct sockaddr_in *usin = (struct sockaddr_in *)uaddr; + struct inet_sock *inet = inet_sk(sk); + struct tcp_sock *tp = tcp_sk(sk); + struct ip_options_rcu *inet_opt; + __be16 orig_sport, orig_dport; + struct net_device *netdev; + struct chtls_dev *cdev; + __be32 daddr, nexthop; + struct flowi4 *fl4; + struct rtable *rt; + int err; + struct inet_timewait_death_row *tcp_death_row = + &sock_net(sk)->ipv4.tcp_death_row; + + if (addr_len < sizeof(struct sockaddr_in)) + return -EINVAL; + + if (usin->sin_family != AF_INET) + return -EAFNOSUPPORT; + + nexthop = usin->sin_addr.s_addr; + daddr = usin->sin_addr.s_addr; + inet_opt = rcu_dereference_protected(inet->inet_opt, + lockdep_sock_is_held(sk)); + if (inet_opt && inet_opt->opt.srr) { + if (!daddr) + return -EINVAL; + + nexthop = inet_opt->opt.faddr; + } + + orig_sport = inet->inet_sport; + orig_dport = usin->sin_port; + fl4 = &inet->cork.fl.u.ip4; + rt = ip_route_connect(fl4, nexthop, inet->inet_saddr, + RT_CONN_FLAGS(sk), sk->sk_bound_dev_if, + IPPROTO_TCP, + orig_sport, orig_dport, sk); + if (IS_ERR(rt)) { + err = PTR_ERR(rt); + if (err == -ENETUNREACH) + IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES); + return err; + } + + if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { + ip_rt_put(rt); + return -ENETUNREACH; + } + + if (!inet_opt || !inet_opt->opt.srr) + daddr = fl4->daddr; + + if (!inet->inet_saddr) + inet->inet_saddr = fl4->saddr; + sk_rcv_saddr_set(sk, inet->inet_saddr); + + if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) { + /* Reset inherited state */ + tp->rx_opt.ts_recent = 0; + tp->rx_opt.ts_recent_stamp = 0; + if (likely(!tp->repair)) + tp->write_seq = 0; + } + + inet->inet_dport = usin->sin_port; + sk_daddr_set(sk, daddr); + + inet_csk(sk)->icsk_ext_hdr_len = 0; + if (inet_opt) + inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen; + + tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT; + + /* Socket identity is still unknown (sport may be zero). + * However we set state to SYN-SENT and not releasing socket + * lock select source port, enter ourselves into the hash tables and + * complete initialization after this. + */ + tcp_set_state(sk, TCP_SYN_SENT); + err = inet_hash_connect(tcp_death_row, sk); + if (err) + goto failure; + + sk_set_txhash(sk); + + rt = ip_route_newports(fl4, rt, orig_sport, orig_dport, + inet->inet_sport, inet->inet_dport, sk); + if (IS_ERR(rt)) { + err = PTR_ERR(rt); + rt = NULL; + goto failure; + } + /* OK, now commit destination to socket. */ + sk->sk_gso_type = SKB_GSO_TCPV4; + sk_setup_caps(sk, &rt->dst); + + cdev = to_chtls_dev(dev); + netdev = __sk_dst_get(sk)->dev; + if (!chtls_ndev_found(cdev, netdev)) { + err = -ENETUNREACH; + rt = NULL; + goto failure; + } + + err = chtls_active_open(cdev, sk, netdev); + if (!err) + return 0; + rt = NULL; + + if (likely(!tp->repair)) { + if (!tp->write_seq) + tp->write_seq = secure_tcp_seq(inet->inet_saddr, + inet->inet_daddr, + inet->inet_sport, + usin->sin_port); + tp->tsoffset = secure_tcp_ts_off(sock_net(sk), + inet->inet_saddr, + inet->inet_daddr); + } + + inet->inet_id = tp->write_seq ^ jiffies; + if (tcp_fastopen_defer_connect(sk, &err)) + return err; + if (err) + goto failure; + + err = tcp_connect(sk); + if (err) + goto failure; + + return 0; +failure: + /* + * This unhashes the socket and releases the local port, + * if necessary. + */ + tcp_set_state(sk, TCP_CLOSE); + ip_rt_put(rt); + sk->sk_route_caps = 0; + inet->inet_dport = 0; + return err; +} + static void chtls_free_uld(struct chtls_dev *cdev) { int i; @@ -195,6 +353,7 @@ static void chtls_register_dev(struct chtls_dev *cdev) tlsdev->feature = chtls_inline_feature; tlsdev->hash = chtls_create_hash; tlsdev->unhash = chtls_destroy_hash; + tlsdev->connect = chtls_connect; tlsdev->release = chtls_dev_release; kref_init(&tlsdev->kref); tls_register_device(tlsdev); @@ -270,6 +429,8 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info) INIT_WORK(&cdev->deferq_task, process_deferq); spin_lock_init(&cdev->listen_lock); spin_lock_init(&cdev->idr_lock); + spin_lock_init(&cdev->aidr_lock); + idr_init(&cdev->aidr); cdev->send_page_order = min_t(uint, get_order(32768), send_page_order); cdev->max_host_sndbuf = 48 * 1024; diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h index 1d9b3e1..796887c 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -158,7 +158,9 @@ struct fw_wr_hdr { /* length in units of 16-bytes (lo) */ #define FW_WR_LEN16_S 0 +#define FW_WR_LEN16_M 0xff #define FW_WR_LEN16_V(x) ((x) << FW_WR_LEN16_S) +#define FW_WR_LEN16_G(x) (((x) >> FW_WR_LEN16_S) & FW_WR_LEN16_M) #define HW_TPL_FR_MT_PR_IV_P_FC 0X32B #define HW_TPL_FR_MT_PR_OV_P_FC 0X327 diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index af6ad46..68cee56 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -123,6 +123,7 @@ u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr) return siphash_2u32((__force u32)saddr, (__force u32)daddr, &ts_secret); } +EXPORT_SYMBOL_GPL(secure_tcp_ts_off); /* secure_tcp_seq_and_tsoff(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d), * but fortunately, `sport' cannot be 0 in any circumstances. If this changes,