@@ -6,6 +6,8 @@ CFLAGS += -I../../../../usr/include/ $(KHDR_INCLUDES)
# Additional include paths needed by kselftest.h
CFLAGS += -I../
+TEST_GEN_MODS_DIR := page_pool
+
TEST_PROGS := run_netsocktests run_afpackettests test_bpf.sh netdevice.sh \
rtnetlink.sh xfrm_policy.sh test_blackhole_dev.sh
TEST_PROGS += fib_tests.sh fib-onlink-tests.sh pmtu.sh udpgso.sh ip_defrag.sh
@@ -96,6 +98,7 @@ TEST_PROGS += fdb_flush.sh
TEST_PROGS += fq_band_pktlimit.sh
TEST_PROGS += vlan_hw_filter.sh
TEST_PROGS += bpf_offload.py
+TEST_PROGS += test_page_pool.sh
TEST_FILES := settings
TEST_FILES += in_netns.sh lib.sh net_helper.sh setup_loopback.sh setup_veth.sh
new file mode 100644
@@ -0,0 +1,18 @@
+PAGE_POOL_TEST_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+KDIR ?= $(abspath $(PAGE_POOL_TEST_DIR)/../../../../..)
+
+ifeq ($(V),1)
+Q =
+else
+Q = @
+endif
+
+MODULES = page_pool_test.ko
+
+obj-m += page_pool_test.o
+
+all:
+ +$(Q)make -C $(KDIR) M=$(PAGE_POOL_TEST_DIR) modules
+
+clean:
+ +$(Q)make -C $(KDIR) M=$(PAGE_POOL_TEST_DIR) clean
new file mode 100644
@@ -0,0 +1,433 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Test module for page_pool
+ *
+ * Copyright (C) 2024 Yunsheng Lin <linyunsheng@huawei.com>
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/etherdevice.h>
+#include <linux/ptr_ring.h>
+#include <linux/kthread.h>
+#include <net/page_pool/helpers.h>
+
+static struct ptr_ring ptr_ring;
+static int nr_objs = 512;
+static atomic_t nthreads;
+static struct completion wait;
+static struct page_pool *test_pool;
+static struct device *dev;
+static u64 dma_mask = DMA_BIT_MASK(64);
+
+static int nr_test = 2000000;
+module_param(nr_test, int, 0);
+MODULE_PARM_DESC(nr_test, "number of iterations to test");
+
+static bool test_frag;
+module_param(test_frag, bool, 0);
+MODULE_PARM_DESC(test_frag, "use frag API for testing");
+
+static bool test_dma;
+module_param(test_dma, bool, 0);
+MODULE_PARM_DESC(test_dma, "enable dma mapping for testing");
+
+static bool test_napi;
+module_param(test_napi, bool, 0);
+MODULE_PARM_DESC(test_napi, "use NAPI softirq for testing");
+
+static bool test_direct;
+module_param(test_direct, bool, 0);
+MODULE_PARM_DESC(test_direct, "enable direct recycle for testing");
+
+static int test_alloc_len = 2048;
+module_param(test_alloc_len, int, 0);
+MODULE_PARM_DESC(test_alloc_len, "alloc len for testing");
+
+static int test_push_cpu;
+module_param(test_push_cpu, int, 0);
+MODULE_PARM_DESC(test_push_cpu, "test cpu for pushing page");
+
+static int test_pop_cpu;
+module_param(test_pop_cpu, int, 0);
+MODULE_PARM_DESC(test_pop_cpu, "test cpu for popping page");
+
+static void page_pool_test_dev_release(struct device *dev)
+{
+ kfree(dev);
+}
+
+static struct page_pool *page_pool_test_create(void)
+{
+ struct page_pool_params page_pool_params = {
+ .pool_size = nr_objs,
+ .flags = 0,
+ .nid = cpu_to_mem(test_push_cpu),
+ };
+ int ret;
+
+ if (test_dma) {
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return ERR_PTR(-ENOMEM);
+
+ dev->release = page_pool_test_dev_release;
+ dev->dma_mask = &dma_mask;
+ device_initialize(dev);
+
+ ret = dev_set_name(dev, "page_pool_dev");
+ if (ret) {
+ pr_err("page_pool_test dev_set_name() failed: %d\n",
+ ret);
+ goto err_out;
+ }
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret) {
+ pr_err("page_pool_test set dma mask failed: %d\n",
+ ret);
+ goto err_out;
+ }
+
+ ret = device_add(dev);
+ if (ret) {
+ pr_err("page_pool_test device_add() failed: %d\n", ret);
+ goto err_out;
+ }
+
+ page_pool_params.dev = dev;
+ page_pool_params.flags |= PP_FLAG_DMA_MAP;
+ page_pool_params.dma_dir = DMA_FROM_DEVICE;
+ }
+
+ return page_pool_create(&page_pool_params);
+err_out:
+ put_device(dev);
+ return ERR_PTR(ret);
+}
+
+static void page_pool_test_destroy(struct page_pool *pool)
+{
+ page_pool_destroy(pool);
+
+ if (test_dma) {
+ device_del(dev);
+ put_device(dev);
+ }
+}
+
+static int test_pushed;
+static int test_popped;
+static int page_pool_pop_thread(void *arg)
+{
+ struct ptr_ring *ring = arg;
+
+ pr_info("page_pool pop test thread begins on cpu %d\n",
+ smp_processor_id());
+
+ while (test_popped < nr_test) {
+ void *obj = __ptr_ring_consume(ring);
+
+ if (obj) {
+ test_popped++;
+ page_pool_put_full_page(test_pool, obj, false);
+ } else {
+ cond_resched();
+ }
+ }
+
+ if (atomic_dec_and_test(&nthreads))
+ complete(&wait);
+
+ pr_info("page_pool pop test thread exits on cpu %d\n",
+ smp_processor_id());
+
+ return 0;
+}
+
+static int page_pool_push_thread(void *arg)
+{
+ struct ptr_ring *ring = arg;
+
+ pr_info("page_pool push test thread begins on cpu %d\n",
+ smp_processor_id());
+
+ while (test_pushed < nr_test) {
+ struct page *page;
+ int ret;
+
+ if (test_frag) {
+ unsigned int offset;
+
+ page = page_pool_dev_alloc_frag(test_pool, &offset,
+ test_alloc_len);
+ } else {
+ page = page_pool_dev_alloc_pages(test_pool);
+ }
+
+ if (!page)
+ continue;
+
+ ret = __ptr_ring_produce(ring, page);
+ if (ret) {
+ page_pool_put_full_page(test_pool, page, true);
+ cond_resched();
+ } else {
+ test_pushed++;
+ }
+ }
+
+ pr_info("page_pool push test thread exits on cpu %d\n",
+ smp_processor_id());
+
+ if (atomic_dec_and_test(&nthreads))
+ complete(&wait);
+
+ return 0;
+}
+
+static int page_pool_push_poll(struct napi_struct *napi, int budget)
+{
+ static bool print = true;
+ int processed = 0;
+
+ if (unlikely(print)) {
+ pr_info("page_pool push test napi begins on cpu %d\n",
+ smp_processor_id());
+ print = false;
+ }
+
+ while (processed < budget && test_pushed < nr_test) {
+ struct page *page;
+ int ret;
+
+ if (test_frag) {
+ unsigned int offset;
+
+ page = page_pool_dev_alloc_frag(test_pool, &offset,
+ test_alloc_len);
+ } else {
+ page = page_pool_dev_alloc_pages(test_pool);
+ }
+
+ if (!page)
+ return budget;
+
+ ret = __ptr_ring_produce(&ptr_ring, page);
+ if (ret) {
+ page_pool_put_full_page(test_pool, page, true);
+ return budget;
+ }
+
+ processed++;
+ test_pushed++;
+ }
+
+ if (test_pushed < nr_test)
+ return budget;
+
+ pr_info("page_pool push test napi exits on cpu %d\n",
+ smp_processor_id());
+
+ napi_complete(napi);
+ if (atomic_dec_and_test(&nthreads))
+ complete(&wait);
+
+ return 0;
+}
+
+static int page_pool_pop_poll(struct napi_struct *napi, int budget)
+{
+ static bool print = true;
+ int processed = 0;
+
+ if (unlikely(print)) {
+ pr_info("page_pool pop test napi begins on cpu %d\n",
+ smp_processor_id());
+ print = false;
+ }
+
+ while (processed < budget && test_popped < nr_test) {
+ void *obj = __ptr_ring_consume(&ptr_ring);
+
+ if (obj) {
+ processed++;
+ test_popped++;
+ page_pool_put_full_page(test_pool, obj, test_direct);
+ } else {
+ return budget;
+ }
+ }
+
+ if (test_popped < nr_test)
+ return budget;
+
+ if (atomic_dec_and_test(&nthreads))
+ complete(&wait);
+
+ napi_complete(napi);
+ pr_info("page_pool pop test napi exits on cpu %d\n",
+ smp_processor_id());
+
+ return 0;
+}
+
+static int page_pool_create_test_thread(void)
+{
+ struct task_struct *tsk_push, *tsk_pop;
+
+ tsk_push = kthread_create_on_cpu(page_pool_push_thread, &ptr_ring,
+ test_push_cpu, "page_pool_push");
+ if (IS_ERR(tsk_push))
+ return PTR_ERR(tsk_push);
+
+ tsk_pop = kthread_create_on_cpu(page_pool_pop_thread, &ptr_ring,
+ test_pop_cpu, "page_pool_pop");
+ if (IS_ERR(tsk_pop)) {
+ kthread_stop(tsk_push);
+ return PTR_ERR(tsk_pop);
+ }
+
+ wake_up_process(tsk_push);
+ wake_up_process(tsk_pop);
+
+ return 0;
+}
+
+static struct napi_struct *pop_napi, *push_napi;
+static struct net_device *netdev;
+static int page_pool_schedule_napi(void *arg)
+{
+ struct napi_struct *napi = arg;
+
+ napi_schedule_irqoff(napi);
+
+ return 0;
+}
+
+static int page_pool_create_test_napi(void)
+{
+ struct task_struct *push_tsk, *pop_tsk;
+ int ret;
+
+ netdev = alloc_etherdev(sizeof(struct napi_struct) * 2);
+ if (!netdev)
+ return -ENOMEM;
+
+ pop_napi = netdev_priv(netdev);
+ push_napi = pop_napi + 1;
+
+ netif_napi_add(netdev, push_napi, page_pool_push_poll);
+ netif_napi_add(netdev, pop_napi, page_pool_pop_poll);
+
+ napi_enable(push_napi);
+ napi_enable(pop_napi);
+
+ push_tsk = kthread_create_on_cpu(page_pool_schedule_napi, push_napi,
+ test_push_cpu, "page_pool_push_napi");
+ if (IS_ERR(push_tsk)) {
+ ret = PTR_ERR(push_tsk);
+ goto err_alloc_etherdev;
+ }
+
+ pop_tsk = kthread_create_on_cpu(page_pool_schedule_napi, pop_napi,
+ test_pop_cpu, "page_pool_pop_napi");
+ if (IS_ERR(pop_tsk)) {
+ ret = PTR_ERR(pop_tsk);
+ goto err_push_thread;
+ }
+
+ wake_up_process(push_tsk);
+ wake_up_process(pop_tsk);
+ return 0;
+
+err_push_thread:
+ kthread_stop(push_tsk);
+err_alloc_etherdev:
+ free_netdev(netdev);
+ return ret;
+}
+
+static void page_pool_destroy_test_napi(void)
+{
+ napi_disable(push_napi);
+ napi_disable(pop_napi);
+
+ netif_napi_del(push_napi);
+ netif_napi_del(pop_napi);
+
+ free_netdev(netdev);
+}
+
+static int __init page_pool_test_init(void)
+{
+ ktime_t start;
+ u64 duration;
+ int ret;
+
+ if (test_alloc_len > PAGE_SIZE || test_alloc_len <= 0 ||
+ !cpu_active(test_push_cpu) || !cpu_active(test_pop_cpu) ||
+ (test_direct && (test_push_cpu != test_pop_cpu || !test_napi)))
+ return -EINVAL;
+
+ ret = ptr_ring_init(&ptr_ring, nr_objs, GFP_KERNEL);
+ if (ret)
+ return ret;
+
+ test_pool = page_pool_test_create();
+ if (IS_ERR(test_pool)) {
+ ret = PTR_ERR(test_pool);
+ goto err_ptr_ring_init;
+ }
+
+ atomic_set(&nthreads, 2);
+ init_completion(&wait);
+
+ if (test_napi)
+ ret = page_pool_create_test_napi();
+ else
+ ret = page_pool_create_test_thread();
+ if (ret)
+ goto err_pool_create;
+
+ start = ktime_get();
+ pr_info("waiting for test to complete\n");
+
+ while (!wait_for_completion_timeout(&wait, msecs_to_jiffies(20000)))
+ pr_info("page_pool_test progress: pushed = %d, popped = %d\n",
+ test_pushed, test_popped);
+
+ duration = (u64)ktime_us_delta(ktime_get(), start);
+ pr_info("%d of iterations for %s%s%s%s testing took: %lluus\n",
+ nr_test, test_napi ? "napi" : "thread",
+ test_direct ? " direct" : "", test_dma ? " dma" : "",
+ test_frag ? " frag" : "", duration);
+
+ ptr_ring_cleanup(&ptr_ring, NULL);
+ page_pool_test_destroy(test_pool);
+
+ if (test_napi)
+ page_pool_destroy_test_napi();
+
+ return -EAGAIN;
+
+err_pool_create:
+ page_pool_test_destroy(test_pool);
+err_ptr_ring_init:
+ ptr_ring_cleanup(&ptr_ring, NULL);
+ return ret;
+}
+
+static void __exit page_pool_test_exit(void)
+{
+}
+
+module_init(page_pool_test_init);
+module_exit(page_pool_test_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Yunsheng Lin <linyunsheng@huawei.com>");
+MODULE_DESCRIPTION("Test module for page_pool");
new file mode 100755
@@ -0,0 +1,175 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2024 Yunsheng Lin <linyunsheng@huawei.com>
+# Copyright (C) 2018 Uladzislau Rezki (Sony) <urezki@gmail.com>
+#
+# This is a test script for the kernel test driver to test the
+# correctness and performance of page_pool's implementation.
+# Therefore it is just a kernel module loader. You can specify
+# and pass different parameters in order to:
+# a) analyse performance of page_pool;
+# b) stressing and stability check of page_pool subsystem.
+
+DRIVER="./page_pool/page_pool_test.ko"
+CPU_LIST=$(grep -m 2 processor /proc/cpuinfo | cut -d ' ' -f 2)
+CPU_CNT=$(echo $CPU_LIST | wc -w)
+TEST_CPU_0=$(echo $CPU_LIST | awk '{print $1}')
+
+if [ $CPU_CNT -gt 1 ]; then
+ TEST_CPU_1=$(echo $CPU_LIST | awk '{print $2}')
+ NR_TEST=100000000
+else
+ TEST_CPU_1=$TEST_CPU_0
+ NR_TEST=1000000
+fi
+
+# Kselftest framework requirement - SKIP code is 4.
+ksft_skip=4
+
+#
+# Static templates for testing of page_pool APIs.
+# Also it is possible to pass any supported parameters manually.
+#
+SMOKE_0_PARAM="test_push_cpu=$TEST_CPU_0 test_pop_cpu=$TEST_CPU_0"
+SMOKE_1_PARAM="test_push_cpu=$TEST_CPU_0 test_pop_cpu=$TEST_CPU_1"
+NONFRAG_PARAM="$SMOKE_1_PARAM nr_test=$NR_TEST"
+FRAG_PARAM="$NONFRAG_PARAM test_alloc_len=2048 test_frag=1"
+NONFRAG_DMA_PARAM="$NONFRAG_PARAM test_dma=1"
+FRAG_DMA_PARAM="$FRAG_PARAM test_dma=1"
+NONFRAG_NAPI_PARAM="$NONFRAG_PARAM test_napi=1"
+FRAG_NAPI_PARAM="$FRAG_PARAM test_napi=1"
+NAPI_PARAM="$SMOKE_0_PARAM test_napi=1"
+NAPI_DIRECT_PARAM="$NAPI_PARAM test_direct=1"
+
+check_test_requirements()
+{
+ uid=$(id -u)
+ if [ $uid -ne 0 ]; then
+ echo "$0: Must be run as root"
+ exit $ksft_skip
+ fi
+
+ if ! which insmod > /dev/null 2>&1; then
+ echo "$0: You need insmod installed"
+ exit $ksft_skip
+ fi
+
+ if [ ! -f $DRIVER ]; then
+ echo "$0: You need to compile page_pool_test module"
+ exit $ksft_skip
+ fi
+}
+
+run_nonfrag_check()
+{
+ echo "Run performance tests to evaluate how fast nonaligned alloc API is."
+
+ insmod $DRIVER $NONFRAG_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_frag_check()
+{
+ echo "Run performance tests to evaluate how fast aligned alloc API is."
+
+ insmod $DRIVER $FRAG_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_nonfrag_dma_check()
+{
+ echo "Run performance tests to evaluate nonaligned alloc API with dma mapping."
+
+ insmod $DRIVER $NONFRAG_DMA_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_frag_dma_check()
+{
+ echo "Run performance tests to evaluate aligned alloc API with dma mapping."
+
+ insmod $DRIVER $FRAG_DMA_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_nonfrag_napi_check()
+{
+ echo "Run performance tests to evaluate nonaligned alloc API in NAPI testing mode."
+
+ insmod $DRIVER $NONFRAG_NAPI_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_frag_napi_check()
+{
+ echo "Run performance tests to evaluate aligned alloc API in NAPI testing mode."
+
+ insmod $DRIVER $FRAG_NAPI_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_napi_check()
+{
+ echo "Run performance in NAPI testing mode."
+
+ insmod $DRIVER $NAPI_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_napi_direct_check()
+{
+ echo "Run performance tests in NAPI and direct recycle testing mode."
+
+ insmod $DRIVER $NAPI_DIRECT_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_smoke_0_check()
+{
+ echo "Run smoke_0 test."
+
+ insmod $DRIVER $SMOKE_0_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+run_smoke_1_check()
+{
+ echo "Run smoke_1 test."
+
+ insmod $DRIVER $SMOKE_1_PARAM > /dev/null 2>&1
+ echo "Done."
+ echo "Check the kernel ring buffer to see the summary."
+}
+
+
+function run_test()
+{
+ run_smoke_0_check
+ run_napi_check
+ run_napi_direct_check
+
+ if [ $CPU_CNT -gt 1 ]; then
+ run_smoke_1_check
+ run_nonfrag_check
+ run_frag_check
+ run_nonfrag_dma_check
+ run_frag_dma_check
+ run_nonfrag_napi_check
+ run_frag_napi_check
+ fi
+}
+
+check_test_requirements
+run_test
+
+exit 0
The testing is done by ensuring that the page allocated from the page_pool instance is pushed into a ptr_ring instance in a kthread/napi binded to a specified cpu, and a kthread/napi binded to a specified cpu will pop the page from the ptr_ring and free it back to the page_pool. Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com> --- tools/testing/selftests/net/Makefile | 3 + .../testing/selftests/net/page_pool/Makefile | 18 + .../selftests/net/page_pool/page_pool_test.c | 433 ++++++++++++++++++ tools/testing/selftests/net/test_page_pool.sh | 175 +++++++ 4 files changed, 629 insertions(+) create mode 100644 tools/testing/selftests/net/page_pool/Makefile create mode 100644 tools/testing/selftests/net/page_pool/page_pool_test.c create mode 100755 tools/testing/selftests/net/test_page_pool.sh