From patchwork Wed Oct 10 19:19:22 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635135 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 27EDD14DB for ; Wed, 10 Oct 2018 19:21:38 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 4A3CF2AA6B for ; Wed, 10 Oct 2018 19:21:38 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 3D9EF2AA77; Wed, 10 Oct 2018 19:21:38 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B4AA32AA4B for ; Wed, 10 Oct 2018 19:21:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727587AbeJKCnk (ORCPT ); Wed, 10 Oct 2018 22:43:40 -0400 Received: from mail.efficios.com ([167.114.142.138]:32792 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727579AbeJKCnj (ORCPT ); Wed, 10 Oct 2018 22:43:39 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id F06FA183B56; Wed, 10 Oct 2018 15:20:05 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id q4DZqpoIpu0W; Wed, 10 Oct 2018 15:20:05 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 2AC14183B53; Wed, 10 Oct 2018 15:20:05 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com 2AC14183B53 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199205; bh=Cfb0if2j7qr27a8wQq79l8o8zLis2u6tHcSE1XnbL0Q=; h=From:To:Date:Message-Id; b=gpjC108iIxTzdhVPF8J7G3bZ7LzBW9DRo6tk6ZTsopfcnZXrsZS2US9xtKLo8ZgJJ Rgb6twi9zHkLtfUk5U6+bNXJykO2o+kPkqZfjbMELhp6awgQc8qkZx8/nvjTv9WrKM EL2/N+i1ylQKroNtgq3A8grSNXu7XC1oZBw1COaq6NO/X3Twhaty7mlUKPmNjS5zMP lLLSrVKwdyg/Ql8IU9+KE7IVFdOsubKdyOf3ix4+MU0ekVi5pHCufe9mKbbr8bjTWD Zc+UWDWuH+RE9h6G7zVjOtowkFl4p82IV+LO6VbiL0FV7RAYZPxslLz1t+E7KkWUfV DOAtCVb7wk2bA== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id tGcDv-7_nAH2; Wed, 10 Oct 2018 15:20:05 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id 006BE183B32; Wed, 10 Oct 2018 15:20:01 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 02/16] rseq/selftests: Adapt number of threads to the number of detected cpus Date: Wed, 10 Oct 2018 15:19:22 -0400 Message-Id: <20181010191936.7495-3-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP On smaller systems, running a test with 200 threads can take a long time on machines with smaller number of CPUs. Detect the number of online cpus at test runtime, and multiply that by 6 to have 6 rseq threads per cpu preempting each other. Signed-off-by: Mathieu Desnoyers Cc: Shuah Khan Cc: Thomas Gleixner Cc: Joel Fernandes Cc: Peter Zijlstra Cc: Catalin Marinas Cc: Dave Watson Cc: Will Deacon Cc: Andi Kleen Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" Cc: Chris Lameter Cc: Russell King Cc: Michael Kerrisk Cc: "Paul E . McKenney" Cc: Paul Turner Cc: Boqun Feng Cc: Josh Triplett Cc: Steven Rostedt Cc: Ben Maurer Cc: Andy Lutomirski Cc: Andrew Morton Cc: Linus Torvalds --- tools/testing/selftests/rseq/run_param_test.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/rseq/run_param_test.sh b/tools/testing/selftests/rseq/run_param_test.sh index 3acd6d75ff9f..e426304fd4a0 100755 --- a/tools/testing/selftests/rseq/run_param_test.sh +++ b/tools/testing/selftests/rseq/run_param_test.sh @@ -1,6 +1,8 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0+ or MIT +NR_CPUS=`grep '^processor' /proc/cpuinfo | wc -l` + EXTRA_ARGS=${@} OLDIFS="$IFS" @@ -28,15 +30,16 @@ IFS="$OLDIFS" REPS=1000 SLOW_REPS=100 +NR_THREADS=$((6*${NR_CPUS})) function do_tests() { local i=0 while [ "$i" -lt "${#TEST_LIST[@]}" ]; do echo "Running test ${TEST_NAME[$i]}" - ./param_test ${TEST_LIST[$i]} -r ${REPS} ${@} ${EXTRA_ARGS} || exit 1 + ./param_test ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 echo "Running compare-twice test ${TEST_NAME[$i]}" - ./param_test_compare_twice ${TEST_LIST[$i]} -r ${REPS} ${@} ${EXTRA_ARGS} || exit 1 + ./param_test_compare_twice ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 let "i++" done } From patchwork Wed Oct 10 19:19:31 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635133 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 604CF16B1 for ; Wed, 10 Oct 2018 19:21:00 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 548012AA2E for ; Wed, 10 Oct 2018 19:21:00 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 486092AA4B; Wed, 10 Oct 2018 19:21:00 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id EA9472AA2E for ; Wed, 10 Oct 2018 19:20:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727395AbeJKCoS (ORCPT ); Wed, 10 Oct 2018 22:44:18 -0400 Received: from mail.efficios.com ([167.114.142.138]:33082 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727279AbeJKCns (ORCPT ); Wed, 10 Oct 2018 22:43:48 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 667E1183C1B; Wed, 10 Oct 2018 15:20:15 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id iIvzeBqiF7kF; Wed, 10 Oct 2018 15:20:14 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id AEA01183C18; Wed, 10 Oct 2018 15:20:14 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com AEA01183C18 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199214; bh=BylZ1Ow4mkEMYbA1e+TyLlzHIq2RSTuuybDdAITGqRU=; h=From:To:Date:Message-Id; b=UsSbQgmpUmDNDV42HU3Fs/GlDavYO5Xrz0tjJMaJ4743S2IpofmTcdQEkSXwiriy0 M8Q5YugaQQ9Grec9EMJNJjnowA6rAvErHq+d2MkpgTP2G+8OlmfyyqoFm2OIPGgxzv X3ljK5FYbd8xEoBRcLyzsPnkuOToY2maGXchTzs6vCNzwCyYYdttnxfs69DaTdX87D yNhiOW8Qd5i2bLy2/m3tXHrEN1cDvVVYkUQsdeMnYR8hyjuLHupR5cPfhXp4MAk1O+ IqHUt2HJMqwpm3hJvpcqknRHL2dpCzjVFmjxn2O7krCc2xIGzPozQwV5r3sGzUL/X2 VIPp78kh+N9hw== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id SyqtElJ7zLld; Wed, 10 Oct 2018 15:20:14 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id A7D1C183BED; Wed, 10 Oct 2018 15:20:13 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 11/16] cpu-opv/selftests: Provide cpu-op library Date: Wed, 10 Oct 2018 15:19:31 -0400 Message-Id: <20181010191936.7495-12-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This cpu-op helper library provides a user-space API to the cpu_opv() system call. Signed-off-by: Mathieu Desnoyers Cc: Thomas Gleixner Cc: Joel Fernandes Cc: Peter Zijlstra Cc: Catalin Marinas Cc: Dave Watson Cc: Will Deacon Cc: Shuah Khan Cc: Andi Kleen Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" Cc: Chris Lameter Cc: Russell King Cc: Michael Kerrisk Cc: "Paul E . McKenney" Cc: Paul Turner Cc: Boqun Feng Cc: Josh Triplett Cc: Steven Rostedt Cc: Ben Maurer Cc: linux-api@vger.kernel.org Cc: Andy Lutomirski Cc: Andrew Morton Cc: Linus Torvalds --- tools/testing/selftests/cpu-opv/cpu-op.c | 353 +++++++++++++++++++++++++++++++ tools/testing/selftests/cpu-opv/cpu-op.h | 42 ++++ 2 files changed, 395 insertions(+) create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.c create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.h diff --git a/tools/testing/selftests/cpu-opv/cpu-op.c b/tools/testing/selftests/cpu-opv/cpu-op.c new file mode 100644 index 000000000000..575cd51da421 --- /dev/null +++ b/tools/testing/selftests/cpu-opv/cpu-op.c @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * cpu-op.c + * + * Copyright (C) 2017 Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; only + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpu-op.h" + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +#define ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) +#define WRITE_ONCE(x, v) __extension__ ({ ACCESS_ONCE(x) = (v); }) +#define READ_ONCE(x) ACCESS_ONCE(x) + +int cpu_opv(struct cpu_op *cpu_opv, int cpuopcnt, int cpu, int flags) +{ + return syscall(__NR_cpu_opv, cpu_opv, cpuopcnt, cpu, flags); +} + +int cpu_op_get_current_cpu(void) +{ + int cpu; + + cpu = sched_getcpu(); + if (cpu < 0) { + perror("sched_getcpu()"); + abort(); + } + return cpu; +} + +int cpu_op_cmpxchg(void *v, void *expect, void *old, void *n, size_t len, + int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)old, + .u.memcpy_op.src = (unsigned long)v, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [1] = { + .op = CPU_COMPARE_EQ_OP, + .len = len, + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [2] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)n, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_add(void *v, int64_t count, size_t len, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_ADD_OP, + .len = len, + .u.arithmetic_op.p = (unsigned long)v, + .u.arithmetic_op.count = count, + .u.arithmetic_op.expect_fault_p = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, + int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +static int cpu_op_cmpeqv_storep_expect_fault(intptr_t *v, intptr_t expect, + intptr_t *newp, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)newp, + .u.memcpy_op.expect_fault_dst = 0, + /* Return EAGAIN on src fault. */ + .u.memcpy_op.expect_fault_src = 1, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu) +{ + int ret; + + do { + intptr_t oldv = READ_ONCE(*v); + intptr_t *newp = (intptr_t *)(oldv + voffp); + + if (oldv == expectnot) + return 1; + ret = cpu_op_cmpeqv_storep_expect_fault(v, oldv, newp, cpu); + if (!ret) { + *load = oldv; + return 0; + } + } while (ret > 0); + + return -1; +} + +int cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v2, + .u.memcpy_op.src = (unsigned long)&newv2, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [2] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpeqv_storev_mb_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v2, + .u.memcpy_op.src = (unsigned long)&newv2, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [2] = { + .op = CPU_MB_OP, + }, + [3] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v2, + .u.compare_op.b = (unsigned long)&expect2, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [2] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [2] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_cmpeqv_memcpy_mb_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = sizeof(intptr_t), + .u.compare_op.a = (unsigned long)v, + .u.compare_op.b = (unsigned long)&expect, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [2] = { + .op = CPU_MB_OP, + }, + [3] = { + .op = CPU_MEMCPY_OP, + .len = sizeof(intptr_t), + .u.memcpy_op.dst = (unsigned long)v, + .u.memcpy_op.src = (unsigned long)&newv, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +int cpu_op_addv(intptr_t *v, int64_t count, int cpu) +{ + return cpu_op_add(v, count, sizeof(intptr_t), cpu); +} diff --git a/tools/testing/selftests/cpu-opv/cpu-op.h b/tools/testing/selftests/cpu-opv/cpu-op.h new file mode 100644 index 000000000000..075687a1365f --- /dev/null +++ b/tools/testing/selftests/cpu-opv/cpu-op.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * cpu-op.h + * + * (C) Copyright 2017-2018 - Mathieu Desnoyers + */ + +#ifndef CPU_OPV_H +#define CPU_OPV_H + +#include +#include +#include + +int cpu_opv(struct cpu_op *cpuopv, int cpuopcnt, int cpu, int flags); +int cpu_op_get_current_cpu(void); + +int cpu_op_cmpxchg(void *v, void *expect, void *old, void *_new, size_t len, + int cpu); +int cpu_op_add(void *v, int64_t count, size_t len, int cpu); + +int cpu_op_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu); +int cpu_op_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu); +int cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu); +int cpu_op_cmpeqv_storev_mb_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu); +int cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu); +int cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu); +int cpu_op_cmpeqv_memcpy_mb_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu); +int cpu_op_addv(intptr_t *v, int64_t count, int cpu); + +#endif /* CPU_OPV_H_ */ From patchwork Wed Oct 10 19:19:32 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635129 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B36C314DB for ; Wed, 10 Oct 2018 19:20:45 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 9FDC02AA2E for ; Wed, 10 Oct 2018 19:20:45 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 93FEB2AA4B; Wed, 10 Oct 2018 19:20:45 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 6D3342AA2E for ; Wed, 10 Oct 2018 19:20:43 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727943AbeJKCnw (ORCPT ); Wed, 10 Oct 2018 22:43:52 -0400 Received: from mail.efficios.com ([167.114.142.138]:33108 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727771AbeJKCnv (ORCPT ); Wed, 10 Oct 2018 22:43:51 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id C9FC7183C31; Wed, 10 Oct 2018 15:20:16 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id lr3tsxPHo4UV; Wed, 10 Oct 2018 15:20:15 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 97590183C23; Wed, 10 Oct 2018 15:20:15 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com 97590183C23 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199215; bh=+9eZK1EkQrNgksmnKzfmvcOHho+8EUQunebQsM+BUjU=; h=From:To:Date:Message-Id; b=s2MhDLrCaOOD55cFAf7SuZS46Vs//nNn9JzPqOpZctaeI5SKOk3shyO7MQ7WLHBjB f2asOFYOomMs4QGQQtwM/cAGfRqkQDO7+3IX3jRfQs4SaJI2Igred4x/mJiBQKFAnH B2qeH0W2P0EsnGfMprmof8HorCzKk5h/kGKxiGWE8nqX5pVR9QQt0sj8yfu5D/J21b 9a9oAbsW6W8cz4UENKTX1FPJhTgyxS93wnF13fCnVheUaPFW5jahRuxJy48kzXhijo VFwA6pqbWMGVWFaWI4koW5SKIh6ZmSV+2BSGlvlMw6y5/f4FzTHNX5q9XfXdoEK2Eq GzKwTDlgdXDDQ== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id hJjrIyv3S_-g; Wed, 10 Oct 2018 15:20:15 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id AC4F7183C17; Wed, 10 Oct 2018 15:20:14 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 12/16] cpu-opv/selftests: Provide basic test Date: Wed, 10 Oct 2018 15:19:32 -0400 Message-Id: <20181010191936.7495-13-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP "basic_cpu_opv_test" tests basic functionality of cpu_opv. Signed-off-by: Mathieu Desnoyers Cc: Thomas Gleixner Cc: Joel Fernandes Cc: Peter Zijlstra Cc: Catalin Marinas Cc: Dave Watson Cc: Will Deacon Cc: Shuah Khan Cc: Andi Kleen Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" Cc: Chris Lameter Cc: Russell King Cc: Michael Kerrisk Cc: "Paul E . McKenney" Cc: Paul Turner Cc: Boqun Feng Cc: Josh Triplett Cc: Steven Rostedt Cc: Ben Maurer Cc: linux-api@vger.kernel.org Cc: Andy Lutomirski Cc: Andrew Morton Cc: Linus Torvalds --- .../testing/selftests/cpu-opv/basic_cpu_opv_test.c | 1362 ++++++++++++++++++++ 1 file changed, 1362 insertions(+) create mode 100644 tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c diff --git a/tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c b/tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c new file mode 100644 index 000000000000..ec5adee20c7f --- /dev/null +++ b/tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c @@ -0,0 +1,1362 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Basic test coverage for cpu_opv system call. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../kselftest.h" + +#include "cpu-op.h" + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +#define TESTBUFLEN 4096 +#define TESTBUFLEN_CMP 16 + +#define TESTBUFLEN_PAGE_MAX 65536 + +#define NR_PF_ARRAY 16384 +#define PF_ARRAY_LEN 4096 + +#define NR_HUGE_ARRAY 512 +#define HUGEMAPLEN (NR_HUGE_ARRAY * PF_ARRAY_LEN) + +/* 64 MB arrays for page fault testing. */ +char pf_array_dst[NR_PF_ARRAY][PF_ARRAY_LEN]; +char pf_array_src[NR_PF_ARRAY][PF_ARRAY_LEN]; + +static int test_ops_supported(void) +{ + const char *test_name = "test_ops_supported"; + int ret; + + ret = cpu_opv(NULL, 0, -1, CPU_OP_NR_FLAG); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret < NR_CPU_OPS) { + ksft_test_result_fail("%s test: only %d operations supported, expecting at least %d\n", + test_name, ret, NR_CPU_OPS); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_compare_eq_op(char *a, char *b, size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = len, + .u.compare_op.a = (unsigned long)a, + .u.compare_op.b = (unsigned long)b, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_compare_eq_same(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_compare_eq same"; + + /* Test compare_eq */ + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + for (i = 0; i < TESTBUFLEN; i++) + buf2[i] = (char)i; + ret = test_compare_eq_op(buf2, buf1, TESTBUFLEN); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret > 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 0); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_compare_eq_diff(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_compare_eq different"; + + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN); + ret = test_compare_eq_op(buf2, buf1, TESTBUFLEN); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret == 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 1); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_compare_ne_op(char *a, char *b, size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_NE_OP, + .len = len, + .u.compare_op.a = (unsigned long)a, + .u.compare_op.b = (unsigned long)b, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_compare_ne_same(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_compare_ne same"; + + /* Test compare_ne */ + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + for (i = 0; i < TESTBUFLEN; i++) + buf2[i] = (char)i; + ret = test_compare_ne_op(buf2, buf1, TESTBUFLEN); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret == 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 1); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_compare_ne_diff(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_compare_ne different"; + + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN); + ret = test_compare_ne_op(buf2, buf1, TESTBUFLEN); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 0); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_2compare_eq_op(char *a, char *b, char *c, char *d, + size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_EQ_OP, + .len = len, + .u.compare_op.a = (unsigned long)a, + .u.compare_op.b = (unsigned long)b, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_COMPARE_EQ_OP, + .len = len, + .u.compare_op.a = (unsigned long)c, + .u.compare_op.b = (unsigned long)d, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_2compare_eq_index(void) +{ + int i, ret; + char buf1[TESTBUFLEN_CMP]; + char buf2[TESTBUFLEN_CMP]; + char buf3[TESTBUFLEN_CMP]; + char buf4[TESTBUFLEN_CMP]; + const char *test_name = "test_2compare_eq index"; + + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN_CMP); + memset(buf3, 0, TESTBUFLEN_CMP); + memset(buf4, 0, TESTBUFLEN_CMP); + + /* First compare failure is op[0], expect 1. */ + ret = test_2compare_eq_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 1) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 1); + return -1; + } + + /* All compares succeed. */ + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf2[i] = (char)i; + ret = test_2compare_eq_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 0); + return -1; + } + + /* First compare failure is op[1], expect 2. */ + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf3[i] = (char)i; + ret = test_2compare_eq_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 2) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 2); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_2compare_ne_op(char *a, char *b, char *c, char *d, + size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_COMPARE_NE_OP, + .len = len, + .u.compare_op.a = (unsigned long)a, + .u.compare_op.b = (unsigned long)b, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + [1] = { + .op = CPU_COMPARE_NE_OP, + .len = len, + .u.compare_op.a = (unsigned long)c, + .u.compare_op.b = (unsigned long)d, + .u.compare_op.expect_fault_a = 0, + .u.compare_op.expect_fault_b = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_2compare_ne_index(void) +{ + int i, ret; + char buf1[TESTBUFLEN_CMP]; + char buf2[TESTBUFLEN_CMP]; + char buf3[TESTBUFLEN_CMP]; + char buf4[TESTBUFLEN_CMP]; + const char *test_name = "test_2compare_ne index"; + + memset(buf1, 0, TESTBUFLEN_CMP); + memset(buf2, 0, TESTBUFLEN_CMP); + memset(buf3, 0, TESTBUFLEN_CMP); + memset(buf4, 0, TESTBUFLEN_CMP); + + /* First compare ne failure is op[0], expect 1. */ + ret = test_2compare_ne_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 1) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 1); + return -1; + } + + /* All compare ne succeed. */ + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf1[i] = (char)i; + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf3[i] = (char)i; + ret = test_2compare_ne_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 0) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 0); + return -1; + } + + /* First compare failure is op[1], expect 2. */ + for (i = 0; i < TESTBUFLEN_CMP; i++) + buf4[i] = (char)i; + ret = test_2compare_ne_op(buf2, buf1, buf4, buf3, TESTBUFLEN_CMP); + if (ret < 0) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret != 2) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, ret, 2); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_memcpy_op(void *dst, void *src, size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_memcpy(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_memcpy"; + + /* Test memcpy */ + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN); + ret = test_memcpy_op(buf2, buf1, TESTBUFLEN); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + for (i = 0; i < TESTBUFLEN; i++) { + if (buf2[i] != (char)i) { + ksft_test_result_fail("%s test: unexpected value at offset %d. Found %d. Should be %d.\n", + test_name, i, buf2[i], (char)i); + return -1; + } + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_memcpy_u32(void) +{ + int ret; + uint32_t v1, v2; + const char *test_name = "test_memcpy_u32"; + + /* Test memcpy_u32 */ + v1 = 42; + v2 = 0; + ret = test_memcpy_op(&v2, &v1, sizeof(v1)); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v1 != v2) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v2, v1); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_memcpy_mb_memcpy_op(void *dst1, void *src1, + void *dst2, void *src2, size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst1, + .u.memcpy_op.src = (unsigned long)src1, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [1] = { + .op = CPU_MB_OP, + }, + [2] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst2, + .u.memcpy_op.src = (unsigned long)src2, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_memcpy_mb_memcpy(void) +{ + int ret; + int v1, v2, v3; + const char *test_name = "test_memcpy_mb_memcpy"; + + /* Test memcpy */ + v1 = 42; + v2 = v3 = 0; + ret = test_memcpy_mb_memcpy_op(&v2, &v1, &v3, &v2, sizeof(int)); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v3 != v1) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v3, v1); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_add_op(int *v, int64_t increment) +{ + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_op_add(v, increment, sizeof(*v), cpu); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_add(void) +{ + int orig_v = 42, v, ret; + int increment = 1; + const char *test_name = "test_add"; + + v = orig_v; + ret = test_add_op(&v, increment); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != orig_v + increment) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, + orig_v + increment); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_two_add_op(int *v, int64_t *increments) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_ADD_OP, + .len = sizeof(*v), + .u.arithmetic_op.p = (unsigned long)v, + .u.arithmetic_op.count = increments[0], + .u.arithmetic_op.expect_fault_p = 0, + }, + [1] = { + .op = CPU_ADD_OP, + .len = sizeof(*v), + .u.arithmetic_op.p = (unsigned long)v, + .u.arithmetic_op.count = increments[1], + .u.arithmetic_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_two_add(void) +{ + int orig_v = 42, v, ret; + int64_t increments[2] = { 99, 123 }; + const char *test_name = "test_two_add"; + + v = orig_v; + ret = test_two_add_op(&v, increments); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != orig_v + increments[0] + increments[1]) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, + orig_v + increments[0] + increments[1]); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_or_op(int *v, uint64_t mask) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_OR_OP, + .len = sizeof(*v), + .u.bitwise_op.p = (unsigned long)v, + .u.bitwise_op.mask = mask, + .u.bitwise_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_or(void) +{ + int orig_v = 0xFF00000, v, ret; + uint32_t mask = 0xFFF; + const char *test_name = "test_or"; + + v = orig_v; + ret = test_or_op(&v, mask); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != (orig_v | mask)) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, orig_v | mask); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_and_op(int *v, uint64_t mask) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_AND_OP, + .len = sizeof(*v), + .u.bitwise_op.p = (unsigned long)v, + .u.bitwise_op.mask = mask, + .u.bitwise_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_and(void) +{ + int orig_v = 0xF00, v, ret; + uint32_t mask = 0xFFF; + const char *test_name = "test_and"; + + v = orig_v; + ret = test_and_op(&v, mask); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != (orig_v & mask)) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, orig_v & mask); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_xor_op(int *v, uint64_t mask) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_XOR_OP, + .len = sizeof(*v), + .u.bitwise_op.p = (unsigned long)v, + .u.bitwise_op.mask = mask, + .u.bitwise_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_xor(void) +{ + int orig_v = 0xF00, v, ret; + uint32_t mask = 0xFFF; + const char *test_name = "test_xor"; + + v = orig_v; + ret = test_xor_op(&v, mask); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != (orig_v ^ mask)) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, orig_v ^ mask); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_lshift_op(int *v, uint32_t bits) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_LSHIFT_OP, + .len = sizeof(*v), + .u.shift_op.p = (unsigned long)v, + .u.shift_op.bits = bits, + .u.shift_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_lshift(void) +{ + int orig_v = 0xF00, v, ret; + uint32_t bits = 5; + const char *test_name = "test_lshift"; + + v = orig_v; + ret = test_lshift_op(&v, bits); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != (orig_v << bits)) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, orig_v << bits); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_rshift_op(int *v, uint32_t bits) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_RSHIFT_OP, + .len = sizeof(*v), + .u.shift_op.p = (unsigned long)v, + .u.shift_op.bits = bits, + .u.shift_op.expect_fault_p = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_rshift(void) +{ + int orig_v = 0xF00, v, ret; + uint32_t bits = 5; + const char *test_name = "test_rshift"; + + v = orig_v; + ret = test_rshift_op(&v, bits); + if (ret) { + ksft_test_result_fail("%s test: returned with %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (v != (orig_v >> bits)) { + ksft_test_result_fail("%s test: unexpected value %d. Should be %d.\n", + test_name, v, orig_v >> bits); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_cmpxchg_op(void *v, void *expect, void *old, void *n, + size_t len) +{ + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_op_cmpxchg(v, expect, old, n, len, cpu); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_cmpxchg_success(void) +{ + int ret; + uint64_t orig_v = 1, v, expect = 1, old = 0, n = 3; + const char *test_name = "test_cmpxchg success"; + + v = orig_v; + ret = test_cmpxchg_op(&v, &expect, &old, &n, sizeof(uint64_t)); + if (ret < 0) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret) { + ksft_test_result_fail("%s returned %d, expecting %d\n", + test_name, ret, 0); + return -1; + } + if (v != n) { + ksft_test_result_fail("%s v is %lld, expecting %lld\n", + test_name, (long long)v, (long long)n); + return -1; + } + if (old != orig_v) { + ksft_test_result_fail("%s old is %lld, expecting %lld\n", + test_name, (long long)old, + (long long)orig_v); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_cmpxchg_fail(void) +{ + int ret; + uint64_t orig_v = 1, v, expect = 123, old = 0, n = 3; + const char *test_name = "test_cmpxchg fail"; + + v = orig_v; + ret = test_cmpxchg_op(&v, &expect, &old, &n, sizeof(uint64_t)); + if (ret < 0) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (ret == 0) { + ksft_test_result_fail("%s returned %d, expecting %d\n", + test_name, ret, 1); + return -1; + } + if (v == n) { + ksft_test_result_fail("%s returned %lld, expecting %lld\n", + test_name, (long long)v, + (long long)orig_v); + return -1; + } + if (old != orig_v) { + ksft_test_result_fail("%s old is %lld, expecting %lld\n", + test_name, (long long)old, + (long long)orig_v); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_memcpy_expect_fault_op(void *dst, void *src, size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + /* Return EAGAIN on fault. */ + .u.memcpy_op.expect_fault_src = 1, + }, + }; + int cpu; + + cpu = cpu_op_get_current_cpu(); + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +static int test_memcpy_fault(void) +{ + int ret; + char buf1[TESTBUFLEN]; + const char *test_name = "test_memcpy_fault"; + + /* Test memcpy */ + ret = test_memcpy_op(buf1, NULL, TESTBUFLEN); + if (!ret || (ret < 0 && errno != EFAULT)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + /* Test memcpy expect fault */ + ret = test_memcpy_expect_fault_op(buf1, NULL, TESTBUFLEN); + if (!ret || (ret < 0 && errno != EAGAIN)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int do_test_unknown_op(void) +{ + struct cpu_op opvec[] = { + [0] = { + .op = -1, /* Unknown */ + .len = 0, + }, + }; + int cpu; + + cpu = cpu_op_get_current_cpu(); + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +static int test_unknown_op(void) +{ + int ret; + const char *test_name = "test_unknown_op"; + + ret = do_test_unknown_op(); + if (!ret || (ret < 0 && errno != EINVAL)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int do_test_max_ops(void) +{ + struct cpu_op opvec[] = { + [0] = { .op = CPU_MB_OP, }, + [1] = { .op = CPU_MB_OP, }, + [2] = { .op = CPU_MB_OP, }, + [3] = { .op = CPU_MB_OP, }, + [4] = { .op = CPU_MB_OP, }, + [5] = { .op = CPU_MB_OP, }, + [6] = { .op = CPU_MB_OP, }, + [7] = { .op = CPU_MB_OP, }, + [8] = { .op = CPU_MB_OP, }, + [9] = { .op = CPU_MB_OP, }, + [10] = { .op = CPU_MB_OP, }, + [11] = { .op = CPU_MB_OP, }, + [12] = { .op = CPU_MB_OP, }, + [13] = { .op = CPU_MB_OP, }, + [14] = { .op = CPU_MB_OP, }, + [15] = { .op = CPU_MB_OP, }, + }; + int cpu; + + cpu = cpu_op_get_current_cpu(); + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +static int test_max_ops(void) +{ + int ret; + const char *test_name = "test_max_ops"; + + ret = do_test_max_ops(); + if (ret < 0) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int do_test_too_many_ops(void) +{ + struct cpu_op opvec[] = { + [0] = { .op = CPU_MB_OP, }, + [1] = { .op = CPU_MB_OP, }, + [2] = { .op = CPU_MB_OP, }, + [3] = { .op = CPU_MB_OP, }, + [4] = { .op = CPU_MB_OP, }, + [5] = { .op = CPU_MB_OP, }, + [6] = { .op = CPU_MB_OP, }, + [7] = { .op = CPU_MB_OP, }, + [8] = { .op = CPU_MB_OP, }, + [9] = { .op = CPU_MB_OP, }, + [10] = { .op = CPU_MB_OP, }, + [11] = { .op = CPU_MB_OP, }, + [12] = { .op = CPU_MB_OP, }, + [13] = { .op = CPU_MB_OP, }, + [14] = { .op = CPU_MB_OP, }, + [15] = { .op = CPU_MB_OP, }, + [16] = { .op = CPU_MB_OP, }, + }; + int cpu; + + cpu = cpu_op_get_current_cpu(); + return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); +} + +static int test_too_many_ops(void) +{ + int ret; + const char *test_name = "test_too_many_ops"; + + ret = do_test_too_many_ops(); + if (!ret || (ret < 0 && errno != EINVAL)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +/* Use 64kB len, largest page size known on Linux. */ +static int test_memcpy_single_too_large(void) +{ + int i, ret; + char buf1[TESTBUFLEN_PAGE_MAX + 1]; + char buf2[TESTBUFLEN_PAGE_MAX + 1]; + const char *test_name = "test_memcpy_single_too_large"; + + /* Test memcpy */ + for (i = 0; i < TESTBUFLEN_PAGE_MAX + 1; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN_PAGE_MAX + 1); + ret = test_memcpy_op(buf2, buf1, TESTBUFLEN_PAGE_MAX + 1); + if (!ret || (ret < 0 && errno != EINVAL)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +static int test_memcpy_single_ok_sum_too_large_op(void *dst, void *src, + size_t len) +{ + struct cpu_op opvec[] = { + [0] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + [1] = { + .op = CPU_MEMCPY_OP, + .len = len, + .u.memcpy_op.dst = (unsigned long)dst, + .u.memcpy_op.src = (unsigned long)src, + .u.memcpy_op.expect_fault_dst = 0, + .u.memcpy_op.expect_fault_src = 0, + }, + }; + int ret, cpu; + + do { + cpu = cpu_op_get_current_cpu(); + ret = cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_memcpy_single_ok_sum_too_large(void) +{ + int i, ret; + char buf1[TESTBUFLEN]; + char buf2[TESTBUFLEN]; + const char *test_name = "test_memcpy_single_ok_sum_too_large"; + + /* Test memcpy */ + for (i = 0; i < TESTBUFLEN; i++) + buf1[i] = (char)i; + memset(buf2, 0, TESTBUFLEN); + ret = test_memcpy_single_ok_sum_too_large_op(buf2, buf1, TESTBUFLEN); + if (!ret || (ret < 0 && errno != EINVAL)) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return -1; + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +/* + * Iterate over large uninitialized arrays to trigger page faults. + * This includes reading from zero pages. + */ +int test_page_fault(void) +{ + int ret = 0; + uint64_t i; + const char *test_name = "test_page_fault"; + + for (i = 0; i < NR_PF_ARRAY; i++) { + ret = test_memcpy_op(pf_array_dst[i], + pf_array_src[i], + PF_ARRAY_LEN); + if (ret) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return ret; + } + } + ksft_test_result_pass("%s test\n", test_name); + return 0; +} + +/* + * Try to use 2MB huge pages. + */ +int test_hugetlb(void) +{ + int ret = 0; + uint64_t i; + const char *test_name = "test_hugetlb"; + int *dst, *src; + + dst = mmap(NULL, HUGEMAPLEN, PROT_READ | PROT_WRITE, + MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (dst == MAP_FAILED) { + switch (errno) { + case ENOMEM: + case ENOENT: + case EINVAL: + ksft_test_result_skip("%s test.\n", test_name); + goto end; + default: + break; + } + perror("mmap"); + abort(); + } + src = mmap(NULL, HUGEMAPLEN, PROT_READ | PROT_WRITE, + MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (src == MAP_FAILED) { + if (errno == ENOMEM) { + ksft_test_result_skip("%s test.\n", test_name); + goto unmap_dst; + } + perror("mmap"); + abort(); + } + + /* Read/write from/to huge zero pages. */ + for (i = 0; i < NR_HUGE_ARRAY; i++) { + ret = test_memcpy_op(dst + (i * PF_ARRAY_LEN / sizeof(int)), + src + (i * PF_ARRAY_LEN / sizeof(int)), + PF_ARRAY_LEN); + if (ret) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return ret; + } + } + for (i = 0; i < NR_HUGE_ARRAY * (PF_ARRAY_LEN / sizeof(int)); i++) + src[i] = i; + + for (i = 0; i < NR_HUGE_ARRAY; i++) { + ret = test_memcpy_op(dst + (i * PF_ARRAY_LEN / sizeof(int)), + src + (i * PF_ARRAY_LEN / sizeof(int)), + PF_ARRAY_LEN); + if (ret) { + ksft_test_result_fail("%s test: ret = %d, errno = %s\n", + test_name, ret, strerror(errno)); + return ret; + } + } + + for (i = 0; i < NR_HUGE_ARRAY * (PF_ARRAY_LEN / sizeof(int)); i++) { + if (dst[i] != i) { + ksft_test_result_fail("%s mismatch, expect %d, got %d\n", + test_name, i, dst[i]); + return ret; + } + } + + ksft_test_result_pass("%s test\n", test_name); + + if (munmap(src, HUGEMAPLEN)) { + perror("munmap"); + abort(); + } +unmap_dst: + if (munmap(dst, HUGEMAPLEN)) { + perror("munmap"); + abort(); + } +end: + return 0; +} + +static int test_cmpxchg_op_cpu(void *v, void *expect, void *old, void *n, + size_t len, int cpu) +{ + int ret; + + do { + ret = cpu_op_cmpxchg(v, expect, old, n, len, cpu); + } while (ret == -1 && errno == EAGAIN); + + return ret; +} + +static int test_over_possible_cpu(void) +{ + int ret; + uint64_t orig_v = 1, v, expect = 1, old = 0, n = 3; + const char *test_name = "test_over_possible_cpu"; + + v = orig_v; + ret = test_cmpxchg_op_cpu(&v, &expect, &old, &n, sizeof(uint64_t), + 0xFFFFFFFF); + if (ret == 0) { + ksft_test_result_fail("%s test: ret = %d\n", + test_name, ret); + return -1; + } + if (ret < 0 && errno == EINVAL) { + ksft_test_result_pass("%s test\n", test_name); + return 0; + } + ksft_test_result_fail("%s returned %d, errno %s, expecting %d, errno %s\n", + test_name, ret, strerror(errno), + 0, strerror(EINVAL)); + return -1; +} + +static int test_allowed_affinity(void) +{ + int ret; + uint64_t orig_v = 1, v, expect = 1, old = 0, n = 3; + const char *test_name = "test_allowed_affinity"; + cpu_set_t allowed_cpus, cpuset; + + ret = sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); + if (ret) { + ksft_test_result_fail("%s returned %d, errno %s\n", + test_name, ret, strerror(errno)); + return -1; + } + if (!(CPU_ISSET(0, &allowed_cpus) && CPU_ISSET(1, &allowed_cpus))) { + ksft_test_result_skip("%s test. Requiring allowed CPUs 0 and 1.\n", + test_name); + return 0; + } + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) { + ksft_test_result_fail("%s test. Unable to set affinity. errno = %s\n", + test_name, strerror(errno)); + return -1; + } + v = orig_v; + ret = test_cmpxchg_op_cpu(&v, &expect, &old, &n, sizeof(uint64_t), + 1); + if (sched_setaffinity(0, sizeof(allowed_cpus), &allowed_cpus) != 0) { + ksft_test_result_fail("%s test. Unable to set affinity. errno = %s\n", + test_name, strerror(errno)); + return -1; + } + if (ret == 0) { + ksft_test_result_fail("%s test: ret = %d\n", + test_name, ret); + return -1; + } + + if (ret < 0 && errno == EINVAL) { + ksft_test_result_pass("%s test\n", test_name); + return 0; + } + ksft_test_result_fail("%s returned %d, errno %s, expecting %d, errno %s\n", + test_name, ret, strerror(errno), + 0, strerror(EINVAL)); + return -1; +} + +int main(int argc, char **argv) +{ + ksft_print_header(); + + test_ops_supported(); + test_compare_eq_same(); + test_compare_eq_diff(); + test_compare_ne_same(); + test_compare_ne_diff(); + test_2compare_eq_index(); + test_2compare_ne_index(); + test_memcpy(); + test_memcpy_u32(); + test_memcpy_mb_memcpy(); + test_add(); + test_two_add(); + test_or(); + test_and(); + test_xor(); + test_lshift(); + test_rshift(); + test_cmpxchg_success(); + test_cmpxchg_fail(); + test_memcpy_fault(); + test_unknown_op(); + test_max_ops(); + test_too_many_ops(); + test_memcpy_single_too_large(); + test_memcpy_single_ok_sum_too_large(); + test_page_fault(); + test_hugetlb(); + test_over_possible_cpu(); + test_allowed_affinity(); + + return ksft_exit_pass(); +} From patchwork Wed Oct 10 19:19:33 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635127 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 2C6AE69B1 for ; Wed, 10 Oct 2018 19:20:41 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 1F8832AA2E for ; Wed, 10 Oct 2018 19:20:41 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 138482AA4B; Wed, 10 Oct 2018 19:20:41 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 55AE52AA2E for ; Wed, 10 Oct 2018 19:20:40 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727771AbeJKCnw (ORCPT ); Wed, 10 Oct 2018 22:43:52 -0400 Received: from mail.efficios.com ([167.114.142.138]:33240 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727874AbeJKCnv (ORCPT ); Wed, 10 Oct 2018 22:43:51 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 139BF183C38; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id umG9iNsW-gsD; Wed, 10 Oct 2018 15:20:16 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 84934183C2D; Wed, 10 Oct 2018 15:20:16 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com 84934183C2D DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199216; bh=/tkEB8pTPAKnqB+zmd5U6i3WoXW9DZMBZszErwJcPFc=; h=From:To:Date:Message-Id; b=rU4dL5OITQtZiaBux+bsRKkDNh7Urb2XrlTYNrYfjcmsmLaJt05qJXKMf7WJaICsy ZGcmiyYoAG4cM+/yLYvRWjzfLra+L7gPjjgF3Rmg8n/4wi0soSzipNCqCFgHb3+kRA wUK/I40QEjBo/nhEJl2OUXEwwfx9nAd64cHWV4htHFWTq9DYobpxEEksJ54KbbKBaV YOFcawd15OoQuYJCcHIDVE8aKHuLgbS0N4nOvbTFWCWUmQY66s9w/So6J12id7S2wX gJKsjfmPN6X6Cs0YBCnUweM2pGSh4fbA1ONT5xb9V/abMdVHDXa2sE0QnAby178Lql Uv8sR4c4B9Nbw== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id xXh-aNnbzbiq; Wed, 10 Oct 2018 15:20:16 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id 94491183C22; Wed, 10 Oct 2018 15:20:15 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 13/16] cpu-opv/selftests: Provide percpu_op API Date: Wed, 10 Oct 2018 15:19:33 -0400 Message-Id: <20181010191936.7495-14-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Introduce percpu-op.h API. It uses rseq internally as fast-path if invoked from the right CPU, else cpu_opv as slow-path if called from the wrong CPU or if rseq fails. This allows acting on per-cpu data from various CPUs transparently from user-space: cpu_opv will take care of migrating the thread to the requested CPU. Use-cases such as rebalancing memory across per-cpu memory pools, or migrating tasks for a user-space scheduler, are thus facilitated. This also handles debugger single-stepping. The use from userspace is, e.g. for a counter increment: int cpu, ret; cpu = percpu_current_cpu(); ret = percpu_addv(&data->c[cpu].count, 1, cpu); if (unlikely(ret)) { perror("percpu_addv"); return -1; } return 0; Signed-off-by: Mathieu Desnoyers CC: Shuah Khan CC: Russell King CC: Catalin Marinas CC: Will Deacon CC: Thomas Gleixner CC: Paul Turner CC: Peter Zijlstra CC: Andy Lutomirski CC: Andi Kleen CC: Dave Watson CC: Chris Lameter CC: Ingo Molnar CC: "H. Peter Anvin" CC: Ben Maurer CC: Steven Rostedt CC: "Paul E. McKenney" CC: Josh Triplett CC: Linus Torvalds CC: Andrew Morton CC: Boqun Feng CC: linux-kselftest@vger.kernel.org CC: linux-api@vger.kernel.org --- tools/testing/selftests/cpu-opv/percpu-op.h | 151 ++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 tools/testing/selftests/cpu-opv/percpu-op.h diff --git a/tools/testing/selftests/cpu-opv/percpu-op.h b/tools/testing/selftests/cpu-opv/percpu-op.h new file mode 100644 index 000000000000..ffc64b268fd3 --- /dev/null +++ b/tools/testing/selftests/cpu-opv/percpu-op.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * percpu-op.h + * + * (C) Copyright 2017-2018 - Mathieu Desnoyers + */ + +#ifndef PERCPU_OP_H +#define PERCPU_OP_H + +#include +#include +#include +#include +#include "rseq.h" +#include "cpu-op.h" + +static inline uint32_t percpu_current_cpu(void) +{ + return rseq_current_cpu(); +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, + int cpu) +{ + int ret; + + ret = rseq_cmpeqv_storev(v, expect, newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev(v, expect, newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu) +{ + int ret; + + ret = rseq_cmpnev_storeoffp_load(v, expectnot, voffp, load, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpnev_storeoffp_load(v, expectnot, voffp, + load, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_addv(intptr_t *v, intptr_t count, int cpu) +{ + if (rseq_unlikely(rseq_addv(v, count, cpu))) + return cpu_op_addv(v, count, cpu); + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev_storev_release(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trystorev_storev_release(v, expect, v2, newv2, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev_mb_storev(v, expect, v2, newv2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_cmpeqv_storev(v, expect, v2, expect2, newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_memcpy_storev_release(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trymemcpy_storev_release(v, expect, dst, src, len, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_memcpy_mb_storev(v, expect, dst, src, len, + newv, cpu); + } + return 0; +} + +#endif /* PERCPU_OP_H_ */ From patchwork Wed Oct 10 19:19:34 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635131 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 1334F14DB for ; Wed, 10 Oct 2018 19:20:59 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 04A122AA34 for ; Wed, 10 Oct 2018 19:20:59 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id EC00C2AA77; Wed, 10 Oct 2018 19:20:58 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2DA7B2AA34 for ; Wed, 10 Oct 2018 19:20:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727132AbeJKCoS (ORCPT ); Wed, 10 Oct 2018 22:44:18 -0400 Received: from mail.efficios.com ([167.114.142.138]:33292 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727875AbeJKCnv (ORCPT ); Wed, 10 Oct 2018 22:43:51 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id DB34B183C53; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id JVp6CrB9PZB6; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id 46432183C48; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com 46432183C48 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199217; bh=gHir4rIYo1hBb6B/d1ojb7dvAjlqUhljmuhdjIhh484=; h=From:To:Date:Message-Id; b=eos8S5Of4pj3YoyzP/cjITnB+ch3XYyZ2lEDZJMu/3TN3hqoHsRCIQO1bUWBYWVSO T1lB97WYx7YwG98c0d0aKVLt4bcvLhyfVLI5cEPkDKd0cUxU9PEEURbVjs/5LegPdr YWQJ+3nZX1NITgHFI8CsaT3pXGAaIFaz6+fdisBpbDyf8WFvakIrR2F7lfkHfRkExX 1SH9p6nLDjIzOWM9j4eUYKbwZtC9EwHs4BVJfhjaBN5LjQRhV2ktFLf+Eum7c9NDHw w4+a98Ej2p2yZpnFQ4o5Za72CDFV1/yIaGTVOUbJGj7R43iy+zjvHmIuxpR0p9PEGH xRJ4ZyK1HSIPg== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id hMlsn5oA2RLJ; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id 82B1A183C2C; Wed, 10 Oct 2018 15:20:16 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 14/16] cpu-opv/selftests: Provide basic percpu ops test Date: Wed, 10 Oct 2018 15:19:34 -0400 Message-Id: <20181010191936.7495-15-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP "basic_percpu_ops_test" implements a few simple per-cpu operations and testing their correctness. Signed-off-by: Mathieu Desnoyers CC: Thomas Gleixner Cc: Joel Fernandes Cc: Peter Zijlstra Cc: Catalin Marinas Cc: Dave Watson Cc: Will Deacon Cc: Shuah Khan Cc: Andi Kleen Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" Cc: Chris Lameter Cc: Russell King Cc: Michael Kerrisk Cc: "Paul E . McKenney" Cc: Paul Turner Cc: Boqun Feng Cc: Josh Triplett Cc: Steven Rostedt Cc: Ben Maurer Cc: linux-api@vger.kernel.org Cc: Andy Lutomirski Cc: Andrew Morton Cc: Linus Torvalds --- .../selftests/cpu-opv/basic_percpu_ops_test.c | 295 +++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 tools/testing/selftests/cpu-opv/basic_percpu_ops_test.c diff --git a/tools/testing/selftests/cpu-opv/basic_percpu_ops_test.c b/tools/testing/selftests/cpu-opv/basic_percpu_ops_test.c new file mode 100644 index 000000000000..2ce5202f25b2 --- /dev/null +++ b/tools/testing/selftests/cpu-opv/basic_percpu_ops_test.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: LGPL-2.1 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#include "percpu-op.h" + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +struct percpu_lock_entry { + intptr_t v; +} __attribute__((aligned(128))); + +struct percpu_lock { + struct percpu_lock_entry c[CPU_SETSIZE]; +}; + +struct test_data_entry { + intptr_t count; +} __attribute__((aligned(128))); + +struct spinlock_test_data { + struct percpu_lock lock; + struct test_data_entry c[CPU_SETSIZE]; + int reps; +}; + +struct percpu_list_node { + intptr_t data; + struct percpu_list_node *next; +}; + +struct percpu_list_entry { + struct percpu_list_node *head; +} __attribute__((aligned(128))); + +struct percpu_list { + struct percpu_list_entry c[CPU_SETSIZE]; +}; + +/* A simple percpu spinlock. */ +void rseq_percpu_lock(struct percpu_lock *lock, int cpu) +{ + for (;;) { + int ret; + + ret = percpu_cmpeqv_storev(&lock->c[cpu].v, + 0, 1, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + /* + * Acquire semantic when taking lock after control dependency. + * Matches rseq_smp_store_release(). + */ + rseq_smp_acquire__after_ctrl_dep(); +} + +void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) +{ + assert(lock->c[cpu].v == 1); + /* + * Release lock, with release semantic. Matches + * rseq_smp_acquire__after_ctrl_dep(). + */ + rseq_smp_store_release(&lock->c[cpu].v, 0); +} + +void *test_percpu_spinlock_thread(void *arg) +{ + struct spinlock_test_data *data = arg; + int i; + + if (rseq_register_current_thread()) { + fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + for (i = 0; i < data->reps; i++) { + int cpu = percpu_current_cpu(); + + rseq_percpu_lock(&data->lock, cpu); + data->c[cpu].count++; + rseq_percpu_unlock(&data->lock, cpu); + } + if (rseq_unregister_current_thread()) { + fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + + return NULL; +} + +/* + * A simple test which implements a sharded counter using a per-cpu + * lock. Obviously real applications might prefer to simply use a + * per-cpu increment; however, this is reasonable for a test and the + * lock can be extended to synchronize more complicated operations. + */ +void test_percpu_spinlock(void) +{ + const int num_threads = 200; + int i; + uint64_t sum; + pthread_t test_threads[num_threads]; + struct spinlock_test_data data; + + memset(&data, 0, sizeof(data)); + data.reps = 5000; + + for (i = 0; i < num_threads; i++) + pthread_create(&test_threads[i], NULL, + test_percpu_spinlock_thread, &data); + + for (i = 0; i < num_threads; i++) + pthread_join(test_threads[i], NULL); + + sum = 0; + for (i = 0; i < CPU_SETSIZE; i++) + sum += data.c[i].count; + + assert(sum == (uint64_t)data.reps * num_threads); +} + +int percpu_list_push(struct percpu_list *list, struct percpu_list_node *node, + int cpu) +{ + for (;;) { + intptr_t *targetptr, newval, expect; + int ret; + + /* Load list->c[cpu].head with single-copy atomicity. */ + expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); + newval = (intptr_t)node; + targetptr = (intptr_t *)&list->c[cpu].head; + node->next = (struct percpu_list_node *)expect; + ret = percpu_cmpeqv_storev(targetptr, expect, newval, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + return cpu; +} + +/* + * Unlike a traditional lock-less linked list; the availability of a + * rseq primitive allows us to implement pop without concerns over + * ABA-type races. + */ +struct percpu_list_node *percpu_list_pop(struct percpu_list *list, + int cpu) +{ + struct percpu_list_node *head; + intptr_t *targetptr, expectnot, *load; + off_t offset; + int ret; + + targetptr = (intptr_t *)&list->c[cpu].head; + expectnot = (intptr_t)NULL; + offset = offsetof(struct percpu_list_node, next); + load = (intptr_t *)&head; + ret = percpu_cmpnev_storeoffp_load(targetptr, expectnot, + offset, load, cpu); + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + if (ret > 0) + return NULL; + return head; +} + +void *test_percpu_list_thread(void *arg) +{ + int i; + struct percpu_list *list = (struct percpu_list *)arg; + + if (rseq_register_current_thread()) { + fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + + for (i = 0; i < 100000; i++) { + struct percpu_list_node *node; + + node = percpu_list_pop(list, percpu_current_cpu()); + sched_yield(); /* encourage shuffling */ + if (node) + percpu_list_push(list, node, percpu_current_cpu()); + } + + if (rseq_unregister_current_thread()) { + fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + + return NULL; +} + +/* Simultaneous modification to a per-cpu linked list from many threads. */ +void test_percpu_list(void) +{ + int i, j; + uint64_t sum = 0, expected_sum = 0; + struct percpu_list list; + pthread_t test_threads[200]; + cpu_set_t allowed_cpus; + + memset(&list, 0, sizeof(list)); + + /* Generate list entries for every usable cpu. */ + sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); + for (i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + for (j = 1; j <= 100; j++) { + struct percpu_list_node *node; + + expected_sum += j; + + node = malloc(sizeof(*node)); + assert(node); + node->data = j; + node->next = list.c[i].head; + list.c[i].head = node; + } + } + + for (i = 0; i < 200; i++) + pthread_create(&test_threads[i], NULL, + test_percpu_list_thread, &list); + + for (i = 0; i < 200; i++) + pthread_join(test_threads[i], NULL); + + for (i = 0; i < CPU_SETSIZE; i++) { + struct percpu_list_node *node; + + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + + while ((node = percpu_list_pop(&list, i))) { + sum += node->data; + free(node); + } + } + + /* + * All entries should now be accounted for (unless some external + * actor is interfering with our allowed affinity while this + * test is running). + */ + assert(sum == expected_sum); +} + +int main(int argc, char **argv) +{ + if (rseq_register_current_thread()) { + fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + goto error; + } + printf("spinlock\n"); + test_percpu_spinlock(); + printf("percpu_list\n"); + test_percpu_list(); + if (rseq_unregister_current_thread()) { + fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + goto error; + } + return 0; + +error: + return -1; +} + From patchwork Wed Oct 10 19:19:35 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Desnoyers X-Patchwork-Id: 10635125 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4AFC816B1 for ; Wed, 10 Oct 2018 19:20:31 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 3C9BA2A9F9 for ; Wed, 10 Oct 2018 19:20:31 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2EFA42AA6B; Wed, 10 Oct 2018 19:20:31 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-8.0 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 4DA7B2A9F9 for ; Wed, 10 Oct 2018 19:20:29 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727996AbeJKCny (ORCPT ); Wed, 10 Oct 2018 22:43:54 -0400 Received: from mail.efficios.com ([167.114.142.138]:33314 "EHLO mail.efficios.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1727877AbeJKCnx (ORCPT ); Wed, 10 Oct 2018 22:43:53 -0400 Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id F3F6A183C61; Wed, 10 Oct 2018 15:20:18 -0400 (EDT) Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10032) with ESMTP id LHAI0d6lvDlT; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from localhost (ip6-localhost [IPv6:::1]) by mail.efficios.com (Postfix) with ESMTP id C72E5183C52; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.efficios.com C72E5183C52 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=efficios.com; s=default; t=1539199217; bh=eGLfT5gYO0I4d7mjTFvY/gLSwB5YKXpBorg1EyXmg/A=; h=From:To:Date:Message-Id; b=FI+nn4vloiv0arxQBn3+Gx6c7JL1eBnLl+cr1uZOi+lXpMCUd2Cy8GfoXVGh8kPXV PEzVthkT+wmeLSclGhEOAy0IV+SygD9+MsaH2d7RSI5gU9TJQEGVgDkde5xYhw/NQB ISr0eS7JbtRVR4u8za/xKU88PfpcsIS+SY3DqFdJfsOh/4UrINwUjJj6OWXBp7SP2+ fYM9qGQBkzXeH0nF6lqnBqvynEPaTjUZtExwE/BCzPYac4OLOsTXBFxoSC3pUBye4O g2vyjygioe/R1Z9+Qbxd6z93Nt6XoBrAmC0Hh1snbLmP3f9v5S1xu8AKLcyidA/cLD GJ9pjy//e9mKg== X-Virus-Scanned: amavisd-new at efficios.com Received: from mail.efficios.com ([IPv6:::1]) by localhost (mail02.efficios.com [IPv6:::1]) (amavisd-new, port 10026) with ESMTP id AAuyontMkLf2; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) Received: from thinkos.internal.efficios.com (192-222-157-41.qc.cable.ebox.net [192.222.157.41]) by mail.efficios.com (Postfix) with ESMTPSA id 3F382183C45; Wed, 10 Oct 2018 15:20:17 -0400 (EDT) From: Mathieu Desnoyers To: Peter Zijlstra , "Paul E . McKenney" , Boqun Feng Cc: linux-kernel@vger.kernel.org, linux-api@vger.kernel.org, Thomas Gleixner , Andy Lutomirski , Dave Watson , Paul Turner , Andrew Morton , Russell King , Ingo Molnar , "H . Peter Anvin" , Andi Kleen , Chris Lameter , Ben Maurer , Steven Rostedt , Josh Triplett , Linus Torvalds , Catalin Marinas , Will Deacon , Michael Kerrisk , Joel Fernandes , Mathieu Desnoyers , Shuah Khan , linux-kselftest@vger.kernel.org Subject: [RFC PATCH for 4.21 15/16] cpu-opv/selftests: Provide parametrized tests Date: Wed, 10 Oct 2018 15:19:35 -0400 Message-Id: <20181010191936.7495-16-mathieu.desnoyers@efficios.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> References: <20181010191936.7495-1-mathieu.desnoyers@efficios.com> Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP "param_test" is a parametrizable percpu operations test using both restartable sequences and cpu_opv. See the "--help" output for usage. "param_test_benchmark" is the same as "param_test", but it removes testing book-keeping code to allow accurate benchmarks. "param_test_compare_twice" is the same as "param_test", but it performs each comparison within rseq critical section twice, thus validating invariants. If any of the second comparisons fails, an error message is printed and the test aborts. Signed-off-by: Mathieu Desnoyers Cc: Thomas Gleixner Cc: Joel Fernandes Cc: Peter Zijlstra Cc: Catalin Marinas Cc: Dave Watson Cc: Will Deacon Cc: Shuah Khan Cc: Andi Kleen Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" Cc: Chris Lameter Cc: Russell King Cc: Michael Kerrisk Cc: "Paul E . McKenney" Cc: Paul Turner Cc: Boqun Feng Cc: Josh Triplett Cc: Steven Rostedt Cc: Ben Maurer Cc: linux-api@vger.kernel.org Cc: Andy Lutomirski Cc: Andrew Morton Cc: Linus Torvalds --- tools/testing/selftests/cpu-opv/param_test.c | 1187 ++++++++++++++++++++++++++ 1 file changed, 1187 insertions(+) create mode 100644 tools/testing/selftests/cpu-opv/param_test.c diff --git a/tools/testing/selftests/cpu-opv/param_test.c b/tools/testing/selftests/cpu-opv/param_test.c new file mode 100644 index 000000000000..c62e75f07385 --- /dev/null +++ b/tools/testing/selftests/cpu-opv/param_test.c @@ -0,0 +1,1187 @@ +// SPDX-License-Identifier: LGPL-2.1 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} + +#define NR_INJECT 9 +static int loop_cnt[NR_INJECT + 1]; + +static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used)); +static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used)); +static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used)); +static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used)); +static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used)); +static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used)); + +static int opt_modulo, verbose; + +static int opt_yield, opt_signal, opt_sleep, + opt_disable_rseq, opt_threads = 200, + opt_disable_mod = 0, opt_test = 's', opt_mb = 0; + +#ifndef RSEQ_SKIP_FASTPATH +static long long opt_reps = 5000; +#else +static long long opt_reps = 100; +#endif + +static __thread __attribute__((tls_model("initial-exec"))) +unsigned int signals_delivered; + +#ifndef BENCHMARK + +static __thread __attribute__((tls_model("initial-exec"), unused)) +unsigned int yield_mod_cnt, nr_abort; + +#define printf_verbose(fmt, ...) \ + do { \ + if (verbose) \ + printf(fmt, ## __VA_ARGS__); \ + } while (0) + +#ifdef __i386__ + +#define INJECT_ASM_REG "eax" + +#define RSEQ_INJECT_CLOBBER \ + , INJECT_ASM_REG + +#define RSEQ_INJECT_ASM(n) \ + "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \ + "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "dec %%" INJECT_ASM_REG "\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" + +#elif defined(__x86_64__) + +#define INJECT_ASM_REG_P "rax" +#define INJECT_ASM_REG "eax" + +#define RSEQ_INJECT_CLOBBER \ + , INJECT_ASM_REG_P \ + , INJECT_ASM_REG + +#define RSEQ_INJECT_ASM(n) \ + "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \ + "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \ + "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "dec %%" INJECT_ASM_REG "\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" + +#elif defined(__ARMEL__) + +#define RSEQ_INJECT_INPUT \ + , [loop_cnt_1]"m"(loop_cnt[1]) \ + , [loop_cnt_2]"m"(loop_cnt[2]) \ + , [loop_cnt_3]"m"(loop_cnt[3]) \ + , [loop_cnt_4]"m"(loop_cnt[4]) \ + , [loop_cnt_5]"m"(loop_cnt[5]) \ + , [loop_cnt_6]"m"(loop_cnt[6]) + +#define INJECT_ASM_REG "r4" + +#define RSEQ_INJECT_CLOBBER \ + , INJECT_ASM_REG + +#define RSEQ_INJECT_ASM(n) \ + "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ + "cmp " INJECT_ASM_REG ", #0\n\t" \ + "beq 333f\n\t" \ + "222:\n\t" \ + "subs " INJECT_ASM_REG ", #1\n\t" \ + "bne 222b\n\t" \ + "333:\n\t" + +#elif __PPC__ + +#define RSEQ_INJECT_INPUT \ + , [loop_cnt_1]"m"(loop_cnt[1]) \ + , [loop_cnt_2]"m"(loop_cnt[2]) \ + , [loop_cnt_3]"m"(loop_cnt[3]) \ + , [loop_cnt_4]"m"(loop_cnt[4]) \ + , [loop_cnt_5]"m"(loop_cnt[5]) \ + , [loop_cnt_6]"m"(loop_cnt[6]) + +#define INJECT_ASM_REG "r18" + +#define RSEQ_INJECT_CLOBBER \ + , INJECT_ASM_REG + +#define RSEQ_INJECT_ASM(n) \ + "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ + "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \ + "beq 333f\n\t" \ + "222:\n\t" \ + "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \ + "bne 222b\n\t" \ + "333:\n\t" +#else +#error unsupported target +#endif + +#define RSEQ_INJECT_FAILED \ + nr_abort++; + +#define RSEQ_INJECT_C(n) \ +{ \ + int loc_i, loc_nr_loops = loop_cnt[n]; \ + \ + for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \ + rseq_barrier(); \ + } \ + if (loc_nr_loops == -1 && opt_modulo) { \ + if (yield_mod_cnt == opt_modulo - 1) { \ + if (opt_sleep > 0) \ + poll(NULL, 0, opt_sleep); \ + if (opt_yield) \ + sched_yield(); \ + if (opt_signal) \ + raise(SIGUSR1); \ + yield_mod_cnt = 0; \ + } else { \ + yield_mod_cnt++; \ + } \ + } \ +} + +#else + +#define printf_verbose(fmt, ...) + +#endif /* BENCHMARK */ + +#include "percpu-op.h" + +struct percpu_lock_entry { + intptr_t v; +} __attribute__((aligned(128))); + +struct percpu_lock { + struct percpu_lock_entry c[CPU_SETSIZE]; +}; + +struct test_data_entry { + intptr_t count; +} __attribute__((aligned(128))); + +struct spinlock_test_data { + struct percpu_lock lock; + struct test_data_entry c[CPU_SETSIZE]; +}; + +struct spinlock_thread_test_data { + struct spinlock_test_data *data; + long long reps; + int reg; +}; + +struct inc_test_data { + struct test_data_entry c[CPU_SETSIZE]; +}; + +struct inc_thread_test_data { + struct inc_test_data *data; + long long reps; + int reg; +}; + +struct percpu_list_node { + intptr_t data; + struct percpu_list_node *next; +}; + +struct percpu_list_entry { + struct percpu_list_node *head; +} __attribute__((aligned(128))); + +struct percpu_list { + struct percpu_list_entry c[CPU_SETSIZE]; +}; + +#define BUFFER_ITEM_PER_CPU 100 + +struct percpu_buffer_node { + intptr_t data; +}; + +struct percpu_buffer_entry { + intptr_t offset; + intptr_t buflen; + struct percpu_buffer_node **array; +} __attribute__((aligned(128))); + +struct percpu_buffer { + struct percpu_buffer_entry c[CPU_SETSIZE]; +}; + +#define MEMCPY_BUFFER_ITEM_PER_CPU 100 + +struct percpu_memcpy_buffer_node { + intptr_t data1; + uint64_t data2; +}; + +struct percpu_memcpy_buffer_entry { + intptr_t offset; + intptr_t buflen; + struct percpu_memcpy_buffer_node *array; +} __attribute__((aligned(128))); + +struct percpu_memcpy_buffer { + struct percpu_memcpy_buffer_entry c[CPU_SETSIZE]; +}; + +/* A simple percpu spinlock. */ +static void rseq_percpu_lock(struct percpu_lock *lock, int cpu) +{ + for (;;) { + int ret; + + ret = percpu_cmpeqv_storev(&lock->c[cpu].v, + 0, 1, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + /* + * Acquire semantic when taking lock after control dependency. + * Matches rseq_smp_store_release(). + */ + rseq_smp_acquire__after_ctrl_dep(); +} + +static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) +{ + assert(lock->c[cpu].v == 1); + /* + * Release lock, with release semantic. Matches + * rseq_smp_acquire__after_ctrl_dep(). + */ + rseq_smp_store_release(&lock->c[cpu].v, 0); +} + +void *test_percpu_spinlock_thread(void *arg) +{ + struct spinlock_thread_test_data *thread_data = arg; + struct spinlock_test_data *data = thread_data->data; + long long i, reps; + + if (!opt_disable_rseq && thread_data->reg && + rseq_register_current_thread()) + abort(); + reps = thread_data->reps; + for (i = 0; i < reps; i++) { + int cpu = rseq_cpu_start(); + + rseq_percpu_lock(&data->lock, cpu); + data->c[cpu].count++; + rseq_percpu_unlock(&data->lock, cpu); +#ifndef BENCHMARK + if (i != 0 && !(i % (reps / 10))) + printf_verbose("tid %d: count %lld\n", (int) gettid(), i); +#endif + } + printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", + (int) gettid(), nr_abort, signals_delivered); + if (!opt_disable_rseq && thread_data->reg && + rseq_unregister_current_thread()) + abort(); + return NULL; +} + +/* + * A simple test which implements a sharded counter using a per-cpu + * lock. Obviously real applications might prefer to simply use a + * per-cpu increment; however, this is reasonable for a test and the + * lock can be extended to synchronize more complicated operations. + */ +void test_percpu_spinlock(void) +{ + const int num_threads = opt_threads; + int i, ret; + uint64_t sum; + pthread_t test_threads[num_threads]; + struct spinlock_test_data data; + struct spinlock_thread_test_data thread_data[num_threads]; + + memset(&data, 0, sizeof(data)); + for (i = 0; i < num_threads; i++) { + thread_data[i].reps = opt_reps; + if (opt_disable_mod <= 0 || (i % opt_disable_mod)) + thread_data[i].reg = 1; + else + thread_data[i].reg = 0; + thread_data[i].data = &data; + ret = pthread_create(&test_threads[i], NULL, + test_percpu_spinlock_thread, + &thread_data[i]); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(test_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + sum = 0; + for (i = 0; i < CPU_SETSIZE; i++) + sum += data.c[i].count; + + assert(sum == (uint64_t)opt_reps * num_threads); +} + +void *test_percpu_inc_thread(void *arg) +{ + struct inc_thread_test_data *thread_data = arg; + struct inc_test_data *data = thread_data->data; + long long i, reps; + + if (!opt_disable_rseq && thread_data->reg && + rseq_register_current_thread()) + abort(); + reps = thread_data->reps; + for (i = 0; i < reps; i++) { + int cpu, ret; + + cpu = rseq_cpu_start(); + ret = percpu_addv(&data->c[cpu].count, 1, cpu); + if (rseq_unlikely(ret)) { + perror("cpu_opv"); + abort(); + } +#ifndef BENCHMARK + if (i != 0 && !(i % (reps / 10))) + printf_verbose("tid %d: count %lld\n", (int) gettid(), i); +#endif + } + printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", + (int) gettid(), nr_abort, signals_delivered); + if (!opt_disable_rseq && thread_data->reg && + rseq_unregister_current_thread()) + abort(); + return NULL; +} + +void test_percpu_inc(void) +{ + const int num_threads = opt_threads; + int i, ret; + uint64_t sum; + pthread_t test_threads[num_threads]; + struct inc_test_data data; + struct inc_thread_test_data thread_data[num_threads]; + + memset(&data, 0, sizeof(data)); + for (i = 0; i < num_threads; i++) { + thread_data[i].reps = opt_reps; + if (opt_disable_mod <= 0 || (i % opt_disable_mod)) + thread_data[i].reg = 1; + else + thread_data[i].reg = 0; + thread_data[i].data = &data; + ret = pthread_create(&test_threads[i], NULL, + test_percpu_inc_thread, + &thread_data[i]); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(test_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + sum = 0; + for (i = 0; i < CPU_SETSIZE; i++) + sum += data.c[i].count; + + assert(sum == (uint64_t)opt_reps * num_threads); +} + +void percpu_list_push(struct percpu_list *list, + struct percpu_list_node *node, + int cpu) +{ + for (;;) { + intptr_t *targetptr, newval, expect; + int ret; + + /* Load list->c[cpu].head with single-copy atomicity. */ + expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); + newval = (intptr_t)node; + targetptr = (intptr_t *)&list->c[cpu].head; + node->next = (struct percpu_list_node *)expect; + ret = percpu_cmpeqv_storev(targetptr, expect, newval, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } +} + +/* + * Unlike a traditional lock-less linked list; the availability of a + * rseq primitive allows us to implement pop without concerns over + * ABA-type races. + */ +struct percpu_list_node *percpu_list_pop(struct percpu_list *list, + int cpu) +{ + struct percpu_list_node *head; + intptr_t *targetptr, expectnot, *load; + off_t offset; + int ret; + + targetptr = (intptr_t *)&list->c[cpu].head; + expectnot = (intptr_t)NULL; + offset = offsetof(struct percpu_list_node, next); + load = (intptr_t *)&head; + ret = percpu_cmpnev_storeoffp_load(targetptr, expectnot, + offset, load, cpu); + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + if (ret > 0) + return NULL; + return head; +} + +void *test_percpu_list_thread(void *arg) +{ + long long i, reps; + struct percpu_list *list = (struct percpu_list *)arg; + + if (!opt_disable_rseq && rseq_register_current_thread()) + abort(); + + reps = opt_reps; + for (i = 0; i < reps; i++) { + struct percpu_list_node *node; + + node = percpu_list_pop(list, rseq_cpu_start()); + if (opt_yield) + sched_yield(); /* encourage shuffling */ + if (node) + percpu_list_push(list, node, rseq_cpu_start()); + } + + printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", + (int) gettid(), nr_abort, signals_delivered); + if (!opt_disable_rseq && rseq_unregister_current_thread()) + abort(); + + return NULL; +} + +/* Simultaneous modification to a per-cpu linked list from many threads. */ +void test_percpu_list(void) +{ + const int num_threads = opt_threads; + int i, j, ret; + uint64_t sum = 0, expected_sum = 0; + struct percpu_list list; + pthread_t test_threads[num_threads]; + cpu_set_t allowed_cpus; + + memset(&list, 0, sizeof(list)); + + /* Generate list entries for every usable cpu. */ + sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); + for (i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + for (j = 1; j <= 100; j++) { + struct percpu_list_node *node; + + expected_sum += j; + + node = malloc(sizeof(*node)); + assert(node); + node->data = j; + node->next = list.c[i].head; + list.c[i].head = node; + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_create(&test_threads[i], NULL, + test_percpu_list_thread, &list); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(test_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + for (i = 0; i < CPU_SETSIZE; i++) { + struct percpu_list_node *node; + + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + + while ((node = percpu_list_pop(&list, i))) { + sum += node->data; + free(node); + } + } + + /* + * All entries should now be accounted for (unless some external + * actor is interfering with our allowed affinity while this + * test is running). + */ + assert(sum == expected_sum); +} + +bool percpu_buffer_push(struct percpu_buffer *buffer, + struct percpu_buffer_node *node, + int cpu) +{ + for (;;) { + intptr_t *targetptr_spec, newval_spec; + intptr_t *targetptr_final, newval_final; + intptr_t offset; + int ret; + + offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); + if (offset == buffer->c[cpu].buflen) + return false; + newval_spec = (intptr_t)node; + targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; + newval_final = offset + 1; + targetptr_final = &buffer->c[cpu].offset; + if (opt_mb) + ret = percpu_cmpeqv_storev_storev_release( + targetptr_final, offset, targetptr_spec, + newval_spec, newval_final, cpu); + else + ret = percpu_cmpeqv_storev_storev(targetptr_final, + offset, targetptr_spec, newval_spec, + newval_final, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + return true; +} + +struct percpu_buffer_node *percpu_buffer_pop(struct percpu_buffer *buffer, + int cpu) +{ + struct percpu_buffer_node *head; + + for (;;) { + intptr_t *targetptr, newval; + intptr_t offset; + int ret; + + /* Load offset with single-copy atomicity. */ + offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); + if (offset == 0) + return NULL; + head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); + newval = offset - 1; + targetptr = (intptr_t *)&buffer->c[cpu].offset; + ret = percpu_cmpeqv_cmpeqv_storev(targetptr, offset, + (intptr_t *)&buffer->c[cpu].array[offset - 1], + (intptr_t)head, newval, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + return head; +} + +void *test_percpu_buffer_thread(void *arg) +{ + long long i, reps; + struct percpu_buffer *buffer = (struct percpu_buffer *)arg; + + if (!opt_disable_rseq && rseq_register_current_thread()) + abort(); + + reps = opt_reps; + for (i = 0; i < reps; i++) { + struct percpu_buffer_node *node; + + node = percpu_buffer_pop(buffer, rseq_cpu_start()); + if (opt_yield) + sched_yield(); /* encourage shuffling */ + if (node) { + if (!percpu_buffer_push(buffer, node, + rseq_cpu_start())) { + /* Should increase buffer size. */ + abort(); + } + } + } + + printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", + (int) gettid(), nr_abort, signals_delivered); + if (!opt_disable_rseq && rseq_unregister_current_thread()) + abort(); + + return NULL; +} + +/* Simultaneous modification to a per-cpu buffer from many threads. */ +void test_percpu_buffer(void) +{ + const int num_threads = opt_threads; + int i, j, ret; + uint64_t sum = 0, expected_sum = 0; + struct percpu_buffer buffer; + pthread_t test_threads[num_threads]; + cpu_set_t allowed_cpus; + + memset(&buffer, 0, sizeof(buffer)); + + /* Generate list entries for every usable cpu. */ + sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); + for (i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + /* Worse-case is every item in same CPU. */ + buffer.c[i].array = + malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * + BUFFER_ITEM_PER_CPU); + assert(buffer.c[i].array); + buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; + for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) { + struct percpu_buffer_node *node; + + expected_sum += j; + + /* + * We could theoretically put the word-sized + * "data" directly in the buffer. However, we + * want to model objects that would not fit + * within a single word, so allocate an object + * for each node. + */ + node = malloc(sizeof(*node)); + assert(node); + node->data = j; + buffer.c[i].array[j - 1] = node; + buffer.c[i].offset++; + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_create(&test_threads[i], NULL, + test_percpu_buffer_thread, &buffer); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(test_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + for (i = 0; i < CPU_SETSIZE; i++) { + struct percpu_buffer_node *node; + + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + + while ((node = percpu_buffer_pop(&buffer, i))) { + sum += node->data; + free(node); + } + free(buffer.c[i].array); + } + + /* + * All entries should now be accounted for (unless some external + * actor is interfering with our allowed affinity while this + * test is running). + */ + assert(sum == expected_sum); +} + +bool percpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, + struct percpu_memcpy_buffer_node item, int cpu) +{ + for (;;) { + intptr_t *targetptr_final, newval_final, offset; + char *destptr, *srcptr; + size_t copylen; + int ret; + + /* Load offset with single-copy atomicity. */ + offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); + if (offset == buffer->c[cpu].buflen) + return false; + destptr = (char *)&buffer->c[cpu].array[offset]; + srcptr = (char *)&item; + /* copylen must be <= 4kB. */ + copylen = sizeof(item); + newval_final = offset + 1; + targetptr_final = &buffer->c[cpu].offset; + if (opt_mb) + ret = percpu_cmpeqv_memcpy_storev_release( + targetptr_final, offset, + destptr, srcptr, copylen, + newval_final, cpu); + else + ret = percpu_cmpeqv_memcpy_storev(targetptr_final, + offset, destptr, srcptr, copylen, + newval_final, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + return true; +} + +bool percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, + struct percpu_memcpy_buffer_node *item, int cpu) +{ + for (;;) { + intptr_t *targetptr_final, newval_final, offset; + char *destptr, *srcptr; + size_t copylen; + int ret; + + /* Load offset with single-copy atomicity. */ + offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); + if (offset == 0) + return false; + destptr = (char *)item; + srcptr = (char *)&buffer->c[cpu].array[offset - 1]; + /* copylen must be <= 4kB. */ + copylen = sizeof(*item); + newval_final = offset - 1; + targetptr_final = &buffer->c[cpu].offset; + ret = percpu_cmpeqv_memcpy_storev(targetptr_final, + offset, destptr, srcptr, copylen, + newval_final, cpu); + if (rseq_likely(!ret)) + break; + if (rseq_unlikely(ret < 0)) { + perror("cpu_opv"); + abort(); + } + /* Retry if comparison fails. */ + } + return true; +} + +void *test_percpu_memcpy_buffer_thread(void *arg) +{ + long long i, reps; + struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; + + if (!opt_disable_rseq && rseq_register_current_thread()) + abort(); + + reps = opt_reps; + for (i = 0; i < reps; i++) { + struct percpu_memcpy_buffer_node item; + bool result; + + result = percpu_memcpy_buffer_pop(buffer, &item, + rseq_cpu_start()); + if (opt_yield) + sched_yield(); /* encourage shuffling */ + if (result) { + if (!percpu_memcpy_buffer_push(buffer, item, + rseq_cpu_start())) { + /* Should increase buffer size. */ + abort(); + } + } + } + + printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", + (int) gettid(), nr_abort, signals_delivered); + if (!opt_disable_rseq && rseq_unregister_current_thread()) + abort(); + + return NULL; +} + +/* Simultaneous modification to a per-cpu buffer from many threads. */ +void test_percpu_memcpy_buffer(void) +{ + const int num_threads = opt_threads; + int i, j, ret; + uint64_t sum = 0, expected_sum = 0; + struct percpu_memcpy_buffer buffer; + pthread_t test_threads[num_threads]; + cpu_set_t allowed_cpus; + + memset(&buffer, 0, sizeof(buffer)); + + /* Generate list entries for every usable cpu. */ + sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); + for (i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + /* Worse-case is every item in same CPU. */ + buffer.c[i].array = + malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * + MEMCPY_BUFFER_ITEM_PER_CPU); + assert(buffer.c[i].array); + buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; + for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) { + expected_sum += 2 * j + 1; + + /* + * We could theoretically put the word-sized + * "data" directly in the buffer. However, we + * want to model objects that would not fit + * within a single word, so allocate an object + * for each node. + */ + buffer.c[i].array[j - 1].data1 = j; + buffer.c[i].array[j - 1].data2 = j + 1; + buffer.c[i].offset++; + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_create(&test_threads[i], NULL, + test_percpu_memcpy_buffer_thread, + &buffer); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(test_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + for (i = 0; i < CPU_SETSIZE; i++) { + struct percpu_memcpy_buffer_node item; + + if (!CPU_ISSET(i, &allowed_cpus)) + continue; + + while (percpu_memcpy_buffer_pop(&buffer, &item, i)) { + sum += item.data1; + sum += item.data2; + } + free(buffer.c[i].array); + } + + /* + * All entries should now be accounted for (unless some external + * actor is interfering with our allowed affinity while this + * test is running). + */ + assert(sum == expected_sum); +} + +static void test_signal_interrupt_handler(int signo) +{ + signals_delivered++; +} + +static int set_signal_handler(void) +{ + int ret = 0; + struct sigaction sa; + sigset_t sigset; + + ret = sigemptyset(&sigset); + if (ret < 0) { + perror("sigemptyset"); + return ret; + } + + sa.sa_handler = test_signal_interrupt_handler; + sa.sa_mask = sigset; + sa.sa_flags = 0; + ret = sigaction(SIGUSR1, &sa, NULL); + if (ret < 0) { + perror("sigaction"); + return ret; + } + + printf_verbose("Signal handler set for SIGUSR1\n"); + + return ret; +} + +static void show_usage(int argc, char **argv) +{ + printf("Usage : %s \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-1 loops] Number of loops for delay injection 1\n"); + printf(" [-2 loops] Number of loops for delay injection 2\n"); + printf(" [-3 loops] Number of loops for delay injection 3\n"); + printf(" [-4 loops] Number of loops for delay injection 4\n"); + printf(" [-5 loops] Number of loops for delay injection 5\n"); + printf(" [-6 loops] Number of loops for delay injection 6\n"); + printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); + printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); + printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); + printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); + printf(" [-y] Yield\n"); + printf(" [-k] Kill thread with signal\n"); + printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); + printf(" [-t N] Number of threads (default 200)\n"); + printf(" [-r N] Number of repetitions per thread (default 5000)\n"); + printf(" [-d] Disable rseq system call (no initialization)\n"); + printf(" [-D M] Disable rseq for each M threads\n"); + printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n"); + printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); + printf(" [-v] Verbose output.\n"); + printf(" [-h] Show this help.\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int i; + + for (i = 1; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); + i++; + break; + case 'm': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_modulo = atol(argv[i + 1]); + if (opt_modulo < 0) { + show_usage(argc, argv); + goto error; + } + i++; + break; + case 's': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_sleep = atol(argv[i + 1]); + if (opt_sleep < 0) { + show_usage(argc, argv); + goto error; + } + i++; + break; + case 'y': + opt_yield = 1; + break; + case 'k': + opt_signal = 1; + break; + case 'd': + opt_disable_rseq = 1; + break; + case 'D': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_disable_mod = atol(argv[i + 1]); + if (opt_disable_mod < 0) { + show_usage(argc, argv); + goto error; + } + i++; + break; + case 't': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_threads = atol(argv[i + 1]); + if (opt_threads < 0) { + show_usage(argc, argv); + goto error; + } + i++; + break; + case 'r': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_reps = atoll(argv[i + 1]); + if (opt_reps < 0) { + show_usage(argc, argv); + goto error; + } + i++; + break; + case 'h': + show_usage(argc, argv); + goto end; + case 'T': + if (argc < i + 2) { + show_usage(argc, argv); + goto error; + } + opt_test = *argv[i + 1]; + switch (opt_test) { + case 's': + case 'l': + case 'i': + case 'b': + case 'm': + break; + default: + show_usage(argc, argv); + goto error; + } + i++; + break; + case 'v': + verbose = 1; + break; + case 'M': + opt_mb = 1; + break; + default: + show_usage(argc, argv); + goto error; + } + } + + loop_cnt_1 = loop_cnt[1]; + loop_cnt_2 = loop_cnt[2]; + loop_cnt_3 = loop_cnt[3]; + loop_cnt_4 = loop_cnt[4]; + loop_cnt_5 = loop_cnt[5]; + loop_cnt_6 = loop_cnt[6]; + + if (set_signal_handler()) + goto error; + + if (!opt_disable_rseq && rseq_register_current_thread()) + goto error; + switch (opt_test) { + case 's': + printf_verbose("spinlock\n"); + test_percpu_spinlock(); + break; + case 'l': + printf_verbose("linked list\n"); + test_percpu_list(); + break; + case 'b': + printf_verbose("buffer\n"); + test_percpu_buffer(); + break; + case 'm': + printf_verbose("memcpy buffer\n"); + test_percpu_memcpy_buffer(); + break; + case 'i': + printf_verbose("counter increment\n"); + test_percpu_inc(); + break; + } + if (!opt_disable_rseq && rseq_unregister_current_thread()) + abort(); +end: + return 0; + +error: + return -1; +}