diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index f9342e21bf31..fd37a61058e7 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -75,6 +75,9 @@
 /* How long an unmatched event or packet can be held */
 #define PKT_EVENT_LIFETIME_MS		10
 
+/* How long unused unicast filters can be held */
+#define UCAST_FILTER_EXPIRY_JIFFIES	msecs_to_jiffies(30000)
+
 /* Offsets into PTP packet for identification.  These offsets are from the
  * start of the IP header, not the MAC header.  Note that neither PTP V1 nor
  * PTP V2 permit the use of IPV4 options.
@@ -226,6 +229,7 @@ struct efx_ptp_rxfilter {
 	__be16 ether_type;
 	__be16 loc_port;
 	__be32 loc_host[4];
+	unsigned long expiry;
 	int handle;
 };
 
@@ -1319,8 +1323,8 @@ static inline void efx_ptp_process_rx(struct efx_nic *efx, struct sk_buff *skb)
 	local_bh_enable();
 }
 
-static bool efx_ptp_filter_exists(struct list_head *ptp_list,
-				  struct efx_filter_spec *spec)
+static struct efx_ptp_rxfilter *
+efx_ptp_find_filter(struct list_head *ptp_list, struct efx_filter_spec *spec)
 {
 	struct efx_ptp_rxfilter *rxfilter;
 
@@ -1328,10 +1332,19 @@ static bool efx_ptp_filter_exists(struct list_head *ptp_list,
 		if (rxfilter->ether_type == spec->ether_type &&
 		    rxfilter->loc_port == spec->loc_port &&
 		    !memcmp(rxfilter->loc_host, spec->loc_host, sizeof(spec->loc_host)))
-			return true;
+			return rxfilter;
 	}
 
-	return false;
+	return NULL;
+}
+
+static inline void efx_ptp_remove_one_filter(struct efx_nic *efx,
+					     struct efx_ptp_rxfilter *rxfilter)
+{
+	efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
+				  rxfilter->handle);
+	list_del(&rxfilter->list);
+	kfree(rxfilter);
 }
 
 static void efx_ptp_remove_filters(struct efx_nic *efx,
@@ -1340,10 +1353,7 @@ static void efx_ptp_remove_filters(struct efx_nic *efx,
 	struct efx_ptp_rxfilter *rxfilter, *tmp;
 
 	list_for_each_entry_safe(rxfilter, tmp, ptp_list, list) {
-		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
-					  rxfilter->handle);
-		list_del(&rxfilter->list);
-		kfree(rxfilter);
+		efx_ptp_remove_one_filter(efx, rxfilter);
 	}
 }
 
@@ -1357,23 +1367,24 @@ static void efx_ptp_init_filter(struct efx_nic *efx,
 			   efx_rx_queue_index(queue));
 }
 
-static int efx_ptp_insert_filter(struct efx_nic *efx,
-				 struct list_head *ptp_list,
-				 struct efx_filter_spec *spec)
+static struct efx_ptp_rxfilter *
+efx_ptp_insert_filter(struct efx_nic *efx, struct list_head *ptp_list,
+		      struct efx_filter_spec *spec)
 {
 	struct efx_ptp_rxfilter *rxfilter;
 	int rc;
 
-	if (efx_ptp_filter_exists(ptp_list, spec))
-		return 0;
+	rxfilter = efx_ptp_find_filter(ptp_list, spec);
+	if (rxfilter)
+		return rxfilter;
 
 	rc = efx_filter_insert_filter(efx, spec, true);
 	if (rc < 0)
-		return rc;
+		return ERR_PTR(rc);
 
 	rxfilter = kzalloc(sizeof(*rxfilter), GFP_KERNEL);
 	if (!rxfilter)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	rxfilter->handle = rc;
 	rxfilter->ether_type = spec->ether_type;
@@ -1381,12 +1392,12 @@ static int efx_ptp_insert_filter(struct efx_nic *efx,
 	memcpy(rxfilter->loc_host, spec->loc_host, sizeof(spec->loc_host));
 	list_add(&rxfilter->list, ptp_list);
 
-	return 0;
+	return rxfilter;
 }
 
-static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx,
-				      struct list_head *ptp_list,
-				      __be32 addr, u16 port)
+static struct efx_ptp_rxfilter *
+efx_ptp_insert_ipv4_filter(struct efx_nic *efx, struct list_head *ptp_list,
+			   __be32 addr, u16 port)
 {
 	struct efx_filter_spec spec;
 
@@ -1395,9 +1406,9 @@ static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx,
 	return efx_ptp_insert_filter(efx, ptp_list, &spec);
 }
 
-static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx,
-				      struct list_head *ptp_list,
-				      struct in6_addr *addr, u16 port)
+static struct efx_ptp_rxfilter *
+efx_ptp_insert_ipv6_filter(struct efx_nic *efx, struct list_head *ptp_list,
+			   struct in6_addr *addr, u16 port)
 {
 	struct efx_filter_spec spec;
 
@@ -1406,7 +1417,8 @@ static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx,
 	return efx_ptp_insert_filter(efx, ptp_list, &spec);
 }
 
-static int efx_ptp_insert_eth_multicast_filter(struct efx_nic *efx)
+static struct efx_ptp_rxfilter *
+efx_ptp_insert_eth_multicast_filter(struct efx_nic *efx)
 {
 	const u8 addr[ETH_ALEN] = PTP_ADDR_ETHER;
 	struct efx_filter_spec spec;
@@ -1421,7 +1433,7 @@ static int efx_ptp_insert_eth_multicast_filter(struct efx_nic *efx)
 static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 {
 	struct efx_ptp_data *ptp = efx->ptp_data;
-	int rc;
+	struct efx_ptp_rxfilter *rc;
 
 	if (!ptp->channel || !list_empty(&ptp->rxfilters_mcast))
 		return 0;
@@ -1431,12 +1443,12 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 	 */
 	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,
 					htonl(PTP_ADDR_IPV4), PTP_EVENT_PORT);
-	if (rc < 0)
+	if (IS_ERR(rc))
 		goto fail;
 
 	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,
 					htonl(PTP_ADDR_IPV4), PTP_GENERAL_PORT);
-	if (rc < 0)
+	if (IS_ERR(rc))
 		goto fail;
 
 	/* if the NIC supports hw timestamps by the MAC, we can support
@@ -1447,16 +1459,16 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 
 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,
 						&ipv6_addr, PTP_EVENT_PORT);
-		if (rc < 0)
+		if (IS_ERR(rc))
 			goto fail;
 
 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,
 						&ipv6_addr, PTP_GENERAL_PORT);
-		if (rc < 0)
+		if (IS_ERR(rc))
 			goto fail;
 
 		rc = efx_ptp_insert_eth_multicast_filter(efx);
-		if (rc < 0)
+		if (IS_ERR(rc))
 			goto fail;
 	}
 
@@ -1464,7 +1476,7 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 
 fail:
 	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);
-	return rc;
+	return PTR_ERR(rc);
 }
 
 static bool efx_ptp_valid_unicast_event_pkt(struct sk_buff *skb)
@@ -1483,7 +1495,7 @@ static int efx_ptp_insert_unicast_filter(struct efx_nic *efx,
 					 struct sk_buff *skb)
 {
 	struct efx_ptp_data *ptp = efx->ptp_data;
-	int rc;
+	struct efx_ptp_rxfilter *rxfilter;
 
 	if (!efx_ptp_valid_unicast_event_pkt(skb))
 		return -EINVAL;
@@ -1491,28 +1503,36 @@ static int efx_ptp_insert_unicast_filter(struct efx_nic *efx,
 	if (skb->protocol == htons(ETH_P_IP)) {
 		__be32 addr = ip_hdr(skb)->saddr;
 
-		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
-						addr, PTP_EVENT_PORT);
-		if (rc < 0)
+		rxfilter = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
+						      addr, PTP_EVENT_PORT);
+		if (IS_ERR(rxfilter))
 			goto fail;
 
-		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
-						addr, PTP_GENERAL_PORT);
-		if (rc < 0)
+		rxfilter->expiry = jiffies + UCAST_FILTER_EXPIRY_JIFFIES;
+
+		rxfilter = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
+						      addr, PTP_GENERAL_PORT);
+		if (rxfilter < 0)
 			goto fail;
+
+		rxfilter->expiry = jiffies + UCAST_FILTER_EXPIRY_JIFFIES;
 	} else if (efx_ptp_use_mac_tx_timestamps(efx)) {
 		/* IPv6 PTP only supported by devices with MAC hw timestamp */
 		struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
 
-		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
-						addr, PTP_EVENT_PORT);
-		if (rc < 0)
+		rxfilter = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
+						      addr, PTP_EVENT_PORT);
+		if (IS_ERR(rxfilter))
 			goto fail;
 
-		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
-						addr, PTP_GENERAL_PORT);
-		if (rc < 0)
+		rxfilter->expiry = jiffies + UCAST_FILTER_EXPIRY_JIFFIES;
+
+		rxfilter = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
+						      addr, PTP_GENERAL_PORT);
+		if (IS_ERR(rxfilter))
 			goto fail;
+
+		rxfilter->expiry = jiffies + UCAST_FILTER_EXPIRY_JIFFIES;
 	} else {
 		return -EOPNOTSUPP;
 	}
@@ -1521,7 +1541,18 @@ static int efx_ptp_insert_unicast_filter(struct efx_nic *efx,
 
 fail:
 	efx_ptp_remove_filters(efx, &ptp->rxfilters_ucast);
-	return rc;
+	return PTR_ERR(rxfilter);
+}
+
+static void efx_ptp_drop_expired_unicast_filters(struct efx_nic *efx)
+{
+	struct efx_ptp_data *ptp = efx->ptp_data;
+	struct efx_ptp_rxfilter *rxfilter, *tmp;
+
+	list_for_each_entry_safe(rxfilter, tmp, &ptp->rxfilters_ucast, list) {
+		if (time_is_before_jiffies(rxfilter->expiry))
+			efx_ptp_remove_one_filter(efx, rxfilter);
+	}
 }
 
 static int efx_ptp_start(struct efx_nic *efx)
@@ -1615,6 +1646,7 @@ static void efx_ptp_worker(struct work_struct *work)
 	}
 
 	efx_ptp_drop_time_expired_events(efx);
+	efx_ptp_drop_expired_unicast_filters(efx);
 
 	__skb_queue_head_init(&tempq);
 	efx_ptp_process_events(efx, &tempq);
