| Message ID | 165765290724.435671.2335548848278684605.stgit@dwillia2-xfh (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | cxl: Region provisioning foundation | expand |
On Tue, 2022-07-12 at 12:08 -0700, Dan Williams wrote: > Exercise the fundamental region provisioning sysfs mechanisms of discovering > available DPA capacity, allocating DPA to a region, and programming HDM > decoders. > > Signed-off-by: Dan Williams <dan.j.williams@intel.com> > --- > test/cxl-region-create.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++ > test/meson.build | 2 + > 2 files changed, 124 insertions(+) > create mode 100644 test/cxl-region-create.sh Since this isn't actually creating a region, should this be named cxl-reserve-dpa.sh ? Alternatively - I guess this test could just be extended to do actual region creation once that is available in cxl-cli? > > diff --git a/test/cxl-region-create.sh b/test/cxl-region-create.sh > new file mode 100644 > index 000000000000..389988759b08 > --- /dev/null > +++ b/test/cxl-region-create.sh > @@ -0,0 +1,122 @@ > +#!/bin/bash > +# SPDX-License-Identifier: GPL-2.0 > +# Copyright (C) 2022 Intel Corporation. All rights reserved. > + > +. $(dirname $0)/common > + > +rc=1 > + > +set -ex > + > +trap 'err $LINENO' ERR > + > +check_prereq "jq" > + > +modprobe -r cxl_test > +modprobe cxl_test > +udevadm settle > + > +# THEORY OF OPERATION: Create a x8 interleave across the pmem capacity > +# of the 8 endpoints defined by cxl_test, commit the decoders (which > +# just stubs out the actual hardware programming aspect, but updates the > +# driver state), and then tear it all down again. As with other cxl_test > +# tests if the CXL topology in tools/testing/cxl/test/cxl.c ever changes > +# then the paired update must be made to this test. > + > +# find the root decoder that spans both test host-bridges and support pmem > +decoder=$(cxl list -b cxl_test -D -d root | jq -r ".[] | This and all following 'cxl' invocations should be "$CXL" which we get from test/common > + select(.pmem_capable == true) | > + select(.nr_targets == 2) | > + .decoder") > + > +# find the memdevs mapped by that decoder > +readarray -t mem < <(cxl list -M -d $decoder | jq -r ".[].memdev") > + > +# ask cxl reserve-dpa to allocate pmem capacity from each of those memdevs > +readarray -t endpoint < <(cxl reserve-dpa -t pmem ${mem[*]} -s $((256<<20)) | > + jq -r ".[] | .decoder.decoder") > + > +# instantiate an empty region > +region=$(cat /sys/bus/cxl/devices/$decoder/create_pmem_region) > +echo $region > /sys/bus/cxl/devices/$decoder/create_pmem_region > +uuidgen > /sys/bus/cxl/devices/$region/uuid > + > +# setup interleave geometry > +nr_targets=${#endpoint[@]} > +echo $nr_targets > /sys/bus/cxl/devices/$region/interleave_ways > +g=$(cat /sys/bus/cxl/devices/$decoder/interleave_granularity) > +echo $g > /sys/bus/cxl/devices/$region/interleave_granularity > +echo $((nr_targets * (256<<20))) > /sys/bus/cxl/devices/$region/size > + > +# grab the list of memdevs grouped by host-bridge interleave position > +port_dev0=$(cxl list -T -d $decoder | jq -r ".[] | > + .targets | .[] | select(.position == 0) | .target") > +port_dev1=$(cxl list -T -d $decoder | jq -r ".[] | > + .targets | .[] | select(.position == 1) | .target") > +readarray -t mem_sort0 < <(cxl list -M -p $port_dev0 | jq -r ".[] | .memdev") > +readarray -t mem_sort1 < <(cxl list -M -p $port_dev1 | jq -r ".[] | .memdev") > + > +# TODO: add a cxl list option to list memdevs in valid region provisioning > +# order, hardcode for now. > +mem_sort=() > +mem_sort[0]=${mem_sort0[0]} > +mem_sort[1]=${mem_sort1[0]} > +mem_sort[2]=${mem_sort0[2]} > +mem_sort[3]=${mem_sort1[2]} > +mem_sort[4]=${mem_sort0[1]} > +mem_sort[5]=${mem_sort1[1]} > +mem_sort[6]=${mem_sort0[3]} > +mem_sort[7]=${mem_sort1[3]} > + > +# TODO: use this alternative memdev ordering to validate a negative test for > +# specifying invalid positions of memdevs > +#mem_sort[2]=${mem_sort0[0]} > +#mem_sort[1]=${mem_sort1[0]} > +#mem_sort[0]=${mem_sort0[2]} > +#mem_sort[3]=${mem_sort1[2]} > +#mem_sort[4]=${mem_sort0[1]} > +#mem_sort[5]=${mem_sort1[1]} > +#mem_sort[6]=${mem_sort0[3]} > +#mem_sort[7]=${mem_sort1[3]} > + > +# re-generate the list of endpoint decoders in region position programming order > +endpoint=() > +for i in ${mem_sort[@]} > +do > + readarray -O ${#endpoint[@]} -t endpoint < <(cxl list -Di -d endpoint -m $i | jq -r ".[] | > + select(.mode == \"pmem\") | .decoder") > +done > + > +# attach all endpoint decoders to the region > +pos=0 > +for i in ${endpoint[@]} > +do > + echo $i > /sys/bus/cxl/devices/$region/target$pos > + pos=$((pos+1)) > +done > +echo "$region added ${#endpoint[@]} targets: ${endpoint[@]}" > + > +# walk up the topology and commit all decoders > +echo 1 > /sys/bus/cxl/devices/$region/commit > + > +# walk down the topology and de-commit all decoders > +echo 0 > /sys/bus/cxl/devices/$region/commit > + > +# remove endpoints from the region > +pos=0 > +for i in ${endpoint[@]} > +do > + echo "" > /sys/bus/cxl/devices/$region/target$pos > + pos=$((pos+1)) > +done > + > +# release DPA capacity > +readarray -t endpoint < <(cxl free-dpa -t pmem ${mem[*]} | > + jq -r ".[] | .decoder.decoder") > +echo "$region released ${#endpoint[@]} targets: ${endpoint[@]}" > + > +# validate no WARN or lockdep report during the run > +log=$(journalctl -r -k --since "-$((SECONDS+1))s") > +grep -q "Call Trace" <<< $log && err "$LINENO" > + > +modprobe -r cxl_test > diff --git a/test/meson.build b/test/meson.build > index 210dcb0b5ff1..fbcfc08d03ee 100644 > --- a/test/meson.build > +++ b/test/meson.build > @@ -151,6 +151,7 @@ max_extent = find_program('max_available_extent_ns.sh') > pfn_meta_errors = find_program('pfn-meta-errors.sh') > track_uuid = find_program('track-uuid.sh') > cxl_topo = find_program('cxl-topology.sh') > +cxl_region = find_program('cxl-region-create.sh') > > tests = [ > [ 'libndctl', libndctl, 'ndctl' ], > @@ -176,6 +177,7 @@ tests = [ > [ 'pfn-meta-errors.sh', pfn_meta_errors, 'ndctl' ], > [ 'track-uuid.sh', track_uuid, 'ndctl' ], > [ 'cxl-topology.sh', cxl_topo, 'cxl' ], > + [ 'cxl-region-create.sh', cxl_region, 'cxl' ], > ] > > if get_option('destructive').enabled() >
Verma, Vishal L wrote: > On Tue, 2022-07-12 at 12:08 -0700, Dan Williams wrote: > > Exercise the fundamental region provisioning sysfs mechanisms of discovering > > available DPA capacity, allocating DPA to a region, and programming HDM > > decoders. > > > > Signed-off-by: Dan Williams <dan.j.williams@intel.com> > > --- > > test/cxl-region-create.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++ > > test/meson.build | 2 + > > 2 files changed, 124 insertions(+) > > create mode 100644 test/cxl-region-create.sh > > Since this isn't actually creating a region, should this be named > cxl-reserve-dpa.sh ? The test goes all the way to the point of registering a new region with libnvdimm, so it is region creation. > Alternatively - I guess this test could just be extended to do actual > region creation once that is available in cxl-cli? I was thinking that's a separate test that moves from just one hardcoded pmem region via sysfs toggling to permuting all the possible cxl_test region configurations across both modes via 'cxl create-region'. One is a sysfs smoke test the other is a create-region unit test. > > > > > diff --git a/test/cxl-region-create.sh b/test/cxl-region-create.sh > > new file mode 100644 > > index 000000000000..389988759b08 > > --- /dev/null > > +++ b/test/cxl-region-create.sh > > @@ -0,0 +1,122 @@ > > +#!/bin/bash > > +# SPDX-License-Identifier: GPL-2.0 > > +# Copyright (C) 2022 Intel Corporation. All rights reserved. > > + > > +. $(dirname $0)/common > > + > > +rc=1 > > + > > +set -ex > > + > > +trap 'err $LINENO' ERR > > + > > +check_prereq "jq" > > + > > +modprobe -r cxl_test > > +modprobe cxl_test > > +udevadm settle > > + > > +# THEORY OF OPERATION: Create a x8 interleave across the pmem capacity > > +# of the 8 endpoints defined by cxl_test, commit the decoders (which > > +# just stubs out the actual hardware programming aspect, but updates the > > +# driver state), and then tear it all down again. As with other cxl_test > > +# tests if the CXL topology in tools/testing/cxl/test/cxl.c ever changes > > +# then the paired update must be made to this test. > > + > > +# find the root decoder that spans both test host-bridges and support pmem > > +decoder=$(cxl list -b cxl_test -D -d root | jq -r ".[] | > > This and all following 'cxl' invocations should be "$CXL" which we get > from test/common Ah, yeah, missed that.
On Wed, 2022-07-13 at 07:47 -0700, Dan Williams wrote: > Verma, Vishal L wrote: > > On Tue, 2022-07-12 at 12:08 -0700, Dan Williams wrote: > > > Exercise the fundamental region provisioning sysfs mechanisms of discovering > > > available DPA capacity, allocating DPA to a region, and programming HDM > > > decoders. > > > > > > Signed-off-by: Dan Williams <dan.j.williams@intel.com> > > > --- > > > test/cxl-region-create.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++ > > > test/meson.build | 2 + > > > 2 files changed, 124 insertions(+) > > > create mode 100644 test/cxl-region-create.sh > > > > Since this isn't actually creating a region, should this be named > > cxl-reserve-dpa.sh ? > > The test goes all the way to the point of registering a new region with > libnvdimm, so it is region creation. > > > Alternatively - I guess this test could just be extended to do actual > > region creation once that is available in cxl-cli? > > I was thinking that's a separate test that moves from just one hardcoded > pmem region via sysfs toggling to permuting all the possible cxl_test > region configurations across both modes via 'cxl create-region'. One is > a sysfs smoke test the other is a create-region unit test. Ah okay makes sense - maybe this should be called cxl-region-sysfs to reflect that? >
Verma, Vishal L wrote: > On Wed, 2022-07-13 at 07:47 -0700, Dan Williams wrote: > > Verma, Vishal L wrote: > > > On Tue, 2022-07-12 at 12:08 -0700, Dan Williams wrote: > > > > Exercise the fundamental region provisioning sysfs mechanisms of discovering > > > > available DPA capacity, allocating DPA to a region, and programming HDM > > > > decoders. > > > > > > > > Signed-off-by: Dan Williams <dan.j.williams@intel.com> > > > > --- > > > > test/cxl-region-create.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++ > > > > test/meson.build | 2 + > > > > 2 files changed, 124 insertions(+) > > > > create mode 100644 test/cxl-region-create.sh > > > > > > Since this isn't actually creating a region, should this be named > > > cxl-reserve-dpa.sh ? > > > > The test goes all the way to the point of registering a new region with > > libnvdimm, so it is region creation. > > > > > Alternatively - I guess this test could just be extended to do actual > > > region creation once that is available in cxl-cli? > > > > I was thinking that's a separate test that moves from just one hardcoded > > pmem region via sysfs toggling to permuting all the possible cxl_test > > region configurations across both modes via 'cxl create-region'. One is > > a sysfs smoke test the other is a create-region unit test. > > Ah okay makes sense - maybe this should be called cxl-region-sysfs to > reflect that? Sure, sounds good.
diff --git a/test/cxl-region-create.sh b/test/cxl-region-create.sh new file mode 100644 index 000000000000..389988759b08 --- /dev/null +++ b/test/cxl-region-create.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2022 Intel Corporation. All rights reserved. + +. $(dirname $0)/common + +rc=1 + +set -ex + +trap 'err $LINENO' ERR + +check_prereq "jq" + +modprobe -r cxl_test +modprobe cxl_test +udevadm settle + +# THEORY OF OPERATION: Create a x8 interleave across the pmem capacity +# of the 8 endpoints defined by cxl_test, commit the decoders (which +# just stubs out the actual hardware programming aspect, but updates the +# driver state), and then tear it all down again. As with other cxl_test +# tests if the CXL topology in tools/testing/cxl/test/cxl.c ever changes +# then the paired update must be made to this test. + +# find the root decoder that spans both test host-bridges and support pmem +decoder=$(cxl list -b cxl_test -D -d root | jq -r ".[] | + select(.pmem_capable == true) | + select(.nr_targets == 2) | + .decoder") + +# find the memdevs mapped by that decoder +readarray -t mem < <(cxl list -M -d $decoder | jq -r ".[].memdev") + +# ask cxl reserve-dpa to allocate pmem capacity from each of those memdevs +readarray -t endpoint < <(cxl reserve-dpa -t pmem ${mem[*]} -s $((256<<20)) | + jq -r ".[] | .decoder.decoder") + +# instantiate an empty region +region=$(cat /sys/bus/cxl/devices/$decoder/create_pmem_region) +echo $region > /sys/bus/cxl/devices/$decoder/create_pmem_region +uuidgen > /sys/bus/cxl/devices/$region/uuid + +# setup interleave geometry +nr_targets=${#endpoint[@]} +echo $nr_targets > /sys/bus/cxl/devices/$region/interleave_ways +g=$(cat /sys/bus/cxl/devices/$decoder/interleave_granularity) +echo $g > /sys/bus/cxl/devices/$region/interleave_granularity +echo $((nr_targets * (256<<20))) > /sys/bus/cxl/devices/$region/size + +# grab the list of memdevs grouped by host-bridge interleave position +port_dev0=$(cxl list -T -d $decoder | jq -r ".[] | + .targets | .[] | select(.position == 0) | .target") +port_dev1=$(cxl list -T -d $decoder | jq -r ".[] | + .targets | .[] | select(.position == 1) | .target") +readarray -t mem_sort0 < <(cxl list -M -p $port_dev0 | jq -r ".[] | .memdev") +readarray -t mem_sort1 < <(cxl list -M -p $port_dev1 | jq -r ".[] | .memdev") + +# TODO: add a cxl list option to list memdevs in valid region provisioning +# order, hardcode for now. +mem_sort=() +mem_sort[0]=${mem_sort0[0]} +mem_sort[1]=${mem_sort1[0]} +mem_sort[2]=${mem_sort0[2]} +mem_sort[3]=${mem_sort1[2]} +mem_sort[4]=${mem_sort0[1]} +mem_sort[5]=${mem_sort1[1]} +mem_sort[6]=${mem_sort0[3]} +mem_sort[7]=${mem_sort1[3]} + +# TODO: use this alternative memdev ordering to validate a negative test for +# specifying invalid positions of memdevs +#mem_sort[2]=${mem_sort0[0]} +#mem_sort[1]=${mem_sort1[0]} +#mem_sort[0]=${mem_sort0[2]} +#mem_sort[3]=${mem_sort1[2]} +#mem_sort[4]=${mem_sort0[1]} +#mem_sort[5]=${mem_sort1[1]} +#mem_sort[6]=${mem_sort0[3]} +#mem_sort[7]=${mem_sort1[3]} + +# re-generate the list of endpoint decoders in region position programming order +endpoint=() +for i in ${mem_sort[@]} +do + readarray -O ${#endpoint[@]} -t endpoint < <(cxl list -Di -d endpoint -m $i | jq -r ".[] | + select(.mode == \"pmem\") | .decoder") +done + +# attach all endpoint decoders to the region +pos=0 +for i in ${endpoint[@]} +do + echo $i > /sys/bus/cxl/devices/$region/target$pos + pos=$((pos+1)) +done +echo "$region added ${#endpoint[@]} targets: ${endpoint[@]}" + +# walk up the topology and commit all decoders +echo 1 > /sys/bus/cxl/devices/$region/commit + +# walk down the topology and de-commit all decoders +echo 0 > /sys/bus/cxl/devices/$region/commit + +# remove endpoints from the region +pos=0 +for i in ${endpoint[@]} +do + echo "" > /sys/bus/cxl/devices/$region/target$pos + pos=$((pos+1)) +done + +# release DPA capacity +readarray -t endpoint < <(cxl free-dpa -t pmem ${mem[*]} | + jq -r ".[] | .decoder.decoder") +echo "$region released ${#endpoint[@]} targets: ${endpoint[@]}" + +# validate no WARN or lockdep report during the run +log=$(journalctl -r -k --since "-$((SECONDS+1))s") +grep -q "Call Trace" <<< $log && err "$LINENO" + +modprobe -r cxl_test diff --git a/test/meson.build b/test/meson.build index 210dcb0b5ff1..fbcfc08d03ee 100644 --- a/test/meson.build +++ b/test/meson.build @@ -151,6 +151,7 @@ max_extent = find_program('max_available_extent_ns.sh') pfn_meta_errors = find_program('pfn-meta-errors.sh') track_uuid = find_program('track-uuid.sh') cxl_topo = find_program('cxl-topology.sh') +cxl_region = find_program('cxl-region-create.sh') tests = [ [ 'libndctl', libndctl, 'ndctl' ], @@ -176,6 +177,7 @@ tests = [ [ 'pfn-meta-errors.sh', pfn_meta_errors, 'ndctl' ], [ 'track-uuid.sh', track_uuid, 'ndctl' ], [ 'cxl-topology.sh', cxl_topo, 'cxl' ], + [ 'cxl-region-create.sh', cxl_region, 'cxl' ], ] if get_option('destructive').enabled()
Exercise the fundamental region provisioning sysfs mechanisms of discovering available DPA capacity, allocating DPA to a region, and programming HDM decoders. Signed-off-by: Dan Williams <dan.j.williams@intel.com> --- test/cxl-region-create.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++ test/meson.build | 2 + 2 files changed, 124 insertions(+) create mode 100644 test/cxl-region-create.sh