| Message ID | 20240624104620.2156041-2-ofir.gal@volumez.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | md: add regression test for "md/md-bitmap: fix writing non bitmap pages" | expand |
On Jun 24, 2024 / 13:46, Ofir Gal wrote: > Move functions from tests/nvme/rc to common/nvme to be able to reuse > them in other tests groups. > > Signed-off-by: Ofir Gal <ofir.gal@volumez.com> > --- > I have moved the function that are nessecary for the regression test I > add. Let me know if we want to move more or all the functions to > common/nvme. > > shellcheck detects 2 new warnings: > common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154] > common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154] > > I have tried to figure out why but I couldn't figure out what is > different than the upstream version. How can I fix them? Hi Ofir, thanks for this work. Please find two comments in line below. One of them suggests a fix for the shellcheck warnings. > > > common/nvme | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++ > tests/nvme/rc | 591 +------------------------------------------------ > 2 files changed, 596 insertions(+), 590 deletions(-) > create mode 100644 common/nvme > > diff --git a/common/nvme b/common/nvme > new file mode 100644 > index 0000000..1800263 > --- /dev/null > +++ b/common/nvme > @@ -0,0 +1,595 @@ > +#!/bin/bash > +# SPDX-License-Identifier: GPL-3.0+ > +# > +# nvme helper functions. > + > +. common/shellcheck > + > +def_traddr="127.0.0.1" > +def_adrfam="ipv4" > +def_trsvcid="4420" > +def_remote_wwnn="0x10001100aa000001" > +def_remote_wwpn="0x20001100aa000001" > +def_local_wwnn="0x10001100aa000002" > +def_local_wwpn="0x20001100aa000002" > +def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" > +def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" > +export def_subsysnqn="blktests-subsystem-1" > +export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" > +_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" > +_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G > +_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 > +nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} > +NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} > +NVMET_CFS="/sys/kernel/config/nvmet/" As for the shellcheck warnings, I suggestto add two lines below here to suppress them. nvme_trtype=${nvme_trtype:-} nvme_adrfam=${nvme_adrfam:-} > + > +# TMPDIR can not be referred out of test() or test_device() context. Instead of > +# global variable def_flie_path, use this getter function. > +_nvme_def_file_path() { > + echo "${TMPDIR}/img" > +} > + > +_require_nvme_trtype() { > + local trtype > + for trtype in "$@"; do > + if [[ "${nvme_trtype}" == "$trtype" ]]; then > + return 0 > + fi > + done > + SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") > + return 1 > +} > + > +_require_nvme_trtype_is_loop() { > + if ! _require_nvme_trtype loop; then > + return 1 > + fi > + return 0 > +} > + > +_require_nvme_trtype_is_fabrics() { > + if ! _require_nvme_trtype loop fc rdma tcp; then > + return 1 > + fi > + return 0 > +} > + > +_nvme_fcloop_add_rport() { > + local local_wwnn="$1" > + local local_wwpn="$2" > + local remote_wwnn="$3" > + local remote_wwpn="$4" > + local loopctl=/sys/class/fcloop/ctl > + > + echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port > +} > + > +_nvme_fcloop_add_lport() { > + local wwnn="$1" > + local wwpn="$2" > + local loopctl=/sys/class/fcloop/ctl > + > + echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port > +} > + > +_nvme_fcloop_add_tport() { > + local wwnn="$1" > + local wwpn="$2" > + local loopctl=/sys/class/fcloop/ctl > + > + echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port > +} > + > +_setup_fcloop() { > + local local_wwnn="${1:-$def_local_wwnn}" > + local local_wwpn="${2:-$def_local_wwpn}" > + local remote_wwnn="${3:-$def_remote_wwnn}" > + local remote_wwpn="${4:-$def_remote_wwpn}" > + > + _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" > + _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" > + _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ > + "${remote_wwnn}" "${remote_wwpn}" > +} > + > +_nvme_fcloop_del_rport() { > + local local_wwnn="$1" > + local local_wwpn="$2" > + local remote_wwnn="$3" > + local remote_wwpn="$4" > + local loopctl=/sys/class/fcloop/ctl > + > + if [[ ! -f "${loopctl}/del_remote_port" ]]; then > + return > + fi > + echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" > +} > + > +_nvme_fcloop_del_lport() { > + local wwnn="$1" > + local wwpn="$2" > + local loopctl=/sys/class/fcloop/ctl > + > + if [[ ! -f "${loopctl}/del_local_port" ]]; then > + return > + fi > + echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" > +} > + > +_nvme_fcloop_del_tport() { > + local wwnn="$1" > + local wwpn="$2" > + local loopctl=/sys/class/fcloop/ctl > + > + if [[ ! -f "${loopctl}/del_target_port" ]]; then > + return > + fi > + echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" > +} > + > +_cleanup_fcloop() { > + local local_wwnn="${1:-$def_local_wwnn}" > + local local_wwpn="${2:-$def_local_wwpn}" > + local remote_wwnn="${3:-$def_remote_wwnn}" > + local remote_wwpn="${4:-$def_remote_wwpn}" > + > + _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" > + _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" > + _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ > + "${remote_wwnn}" "${remote_wwpn}" > +} > + > +_cleanup_blkdev() { > + local blkdev > + local dev > + > + blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" > + for dev in ${blkdev}; do > + losetup -d "${dev}" > + done > + rm -f "$(_nvme_def_file_path)" > +} > + > +_cleanup_nvmet() { > + local dev > + local port > + local subsys > + local transport > + local name > + > + if [[ ! -d "${NVMET_CFS}" ]]; then > + return 0 > + fi > + > + # Don't let successive Ctrl-Cs interrupt the cleanup processes > + trap '' SIGINT > + > + shopt -s nullglob > + > + for dev in /sys/class/nvme/nvme*; do > + dev="$(basename "$dev")" > + transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" > + if [[ "$transport" == "${nvme_trtype}" ]]; then > + # if udev auto connect is enabled for FC we get false positives > + if [[ "$transport" != "fc" ]]; then > + echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" > + fi > + _nvme_disconnect_ctrl "${dev}" 2>/dev/null > + fi > + done > + > + for port in "${NVMET_CFS}"/ports/*; do > + name=$(basename "${port}") > + echo "WARNING: Test did not clean up port: ${name}" > + rm -f "${port}"/subsystems/* > + rmdir "${port}" > + done > + > + for subsys in "${NVMET_CFS}"/subsystems/*; do > + name=$(basename "${subsys}") > + echo "WARNING: Test did not clean up subsystem: ${name}" > + for ns in "${subsys}"/namespaces/*; do > + rmdir "${ns}" > + done > + rmdir "${subsys}" > + done > + > + for host in "${NVMET_CFS}"/hosts/*; do > + name=$(basename "${host}") > + echo "WARNING: Test did not clean up host: ${name}" > + rmdir "${host}" > + done > + > + shopt -u nullglob > + trap SIGINT > + > + if [[ "${nvme_trtype}" == "fc" ]]; then > + _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ > + "${def_remote_wwnn}" "${def_remote_wwpn}" > + modprobe -rq nvme-fcloop 2>/dev/null > + fi > + modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null > + if [[ "${nvme_trtype}" != "loop" ]]; then > + modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null > + fi > + modprobe -rq nvmet 2>/dev/null > + if [[ "${nvme_trtype}" == "rdma" ]]; then > + stop_soft_rdma > + fi > + > + _cleanup_blkdev > +} > + > +_setup_nvmet() { > + _register_test_cleanup _cleanup_nvmet > + modprobe -q nvmet > + if [[ "${nvme_trtype}" != "loop" ]]; then > + modprobe -q nvmet-"${nvme_trtype}" > + fi > + modprobe -q nvme-"${nvme_trtype}" > + if [[ "${nvme_trtype}" == "rdma" ]]; then > + start_soft_rdma > + for i in $(rdma_network_interfaces) > + do > + if [[ "${nvme_adrfam}" == "ipv6" ]]; then > + ipv6_addr=$(get_ipv6_ll_addr "$i") > + if [[ -n "${ipv6_addr}" ]]; then > + def_traddr=${ipv6_addr} > + fi > + else > + ipv4_addr=$(get_ipv4_addr "$i") > + if [[ -n "${ipv4_addr}" ]]; then > + def_traddr=${ipv4_addr} > + fi > + fi > + done > + fi > + if [[ "${nvme_trtype}" = "fc" ]]; then > + modprobe -q nvme-fcloop > + _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ > + "${def_remote_wwnn}" "${def_remote_wwpn}" > + > + def_traddr=$(printf "nn-%s:pn-%s" \ > + "${def_remote_wwnn}" \ > + "${def_remote_wwpn}") > + def_host_traddr=$(printf "nn-%s:pn-%s" \ > + "${def_local_wwnn}" \ > + "${def_local_wwpn}") > + fi > +} > + > +_nvme_disconnect_ctrl() { > + local ctrl="$1" > + > + nvme disconnect --device "${ctrl}" > +} > + > +_nvme_connect_subsys() { > + local subsysnqn="$def_subsysnqn" > + local hostnqn="$def_hostnqn" > + local hostid="$def_hostid" > + local hostkey="" > + local ctrlkey="" > + local nr_io_queues="" > + local nr_write_queues="" > + local nr_poll_queues="" > + local keep_alive_tmo="" > + local reconnect_delay="" > + local ctrl_loss_tmo="" > + local no_wait=false > + local i > + > + while [[ $# -gt 0 ]]; do > + case $1 in > + --subsysnqn) > + subsysnqn="$2" > + shift 2 > + ;; > + --hostnqn) > + hostnqn="$2" > + shift 2 > + ;; > + --hostid) > + hostid="$2" > + shift 2 > + ;; > + --dhchap-secret) > + hostkey="$2" > + shift 2 > + ;; > + --dhchap-ctrl-secret) > + ctrlkey="$2" > + shift 2 > + ;; > + --nr-io-queues) > + nr_io_queues="$2" > + shift 2 > + ;; > + --nr-write-queues) > + nr_write_queues="$2" > + shift 2 > + ;; > + --nr-poll-queues) > + nr_poll_queues="$2" > + shift 2 > + ;; > + --keep-alive-tmo) > + keep_alive_tmo="$2" > + shift 2 > + ;; > + --reconnect-delay) > + reconnect_delay="$2" > + shift 2 > + ;; > + --ctrl-loss-tmo) > + ctrl_loss_tmo="$2" > + shift 2 > + ;; > + --no-wait) > + no_wait=true > + shift 1 > + ;; > + *) > + echo "WARNING: unknown argument: $1" > + shift > + ;; > + esac > + done > + > + ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") > + if [[ "${nvme_trtype}" == "fc" ]] ; then > + ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") > + elif [[ "${nvme_trtype}" != "loop" ]]; then > + ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") > + fi > + ARGS+=(--hostnqn="${hostnqn}") > + ARGS+=(--hostid="${hostid}") > + if [[ -n "${hostkey}" ]]; then > + ARGS+=(--dhchap-secret="${hostkey}") > + fi > + if [[ -n "${ctrlkey}" ]]; then > + ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") > + fi > + if [[ -n "${nr_io_queues}" ]]; then > + ARGS+=(--nr-io-queues="${nr_io_queues}") > + fi > + if [[ -n "${nr_write_queues}" ]]; then > + ARGS+=(--nr-write-queues="${nr_write_queues}") > + fi > + if [[ -n "${nr_poll_queues}" ]]; then > + ARGS+=(--nr-poll-queues="${nr_poll_queues}") > + fi > + if [[ -n "${keep_alive_tmo}" ]]; then > + ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") > + fi > + if [[ -n "${reconnect_delay}" ]]; then > + ARGS+=(--reconnect-delay="${reconnect_delay}") > + fi > + if [[ -n "${ctrl_loss_tmo}" ]]; then > + ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") > + fi > + > + nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" > + > + # Wait until device file and uuid/wwid sysfs attributes get ready for > + # all namespaces. > + if [[ ${no_wait} = false ]]; then > + udevadm settle > + for ((i = 0; i < 10; i++)); do > + _nvme_ns_ready "${subsysnqn}" && return > + sleep .1 > + done > + fi > +} > + > +_nvme_ns_ready() { > + local subsysnqn="${1}" > + local ns_path ns_id dev > + local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" > + > + dev=$(_find_nvme_dev "$subsysnqn") > + for ns_path in "${cfs_path}/namespaces/"*; do > + ns_id=${ns_path##*/} > + if [[ ! -b /dev/${dev}n${ns_id} || > + ! -e /sys/block/${dev}n${ns_id}/uuid || > + ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then > + return 1 > + fi > + done > + return 0 > +} > + > +_create_nvmet_port() { > + local trtype="$1" > + local traddr="${2:-$def_traddr}" > + local adrfam="${3:-$def_adrfam}" > + local trsvcid="${4:-$def_trsvcid}" > + > + local port > + for ((port = 0; ; port++)); do > + if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then > + break > + fi > + done > + > + mkdir "${NVMET_CFS}/ports/${port}" > + echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" > + echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" > + echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" > + if [[ "${adrfam}" != "fc" ]]; then > + echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" > + fi > + > + echo "${port}" > +} > + > +_remove_nvmet_port() { > + local port="$1" > + rmdir "${NVMET_CFS}/ports/${port}" > +} > + > +_create_nvmet_ns() { > + local nvmet_subsystem="$1" > + local nsid="$2" > + local blkdev="$3" > + local uuid="00000000-0000-0000-0000-000000000000" > + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > + local ns_path="${subsys_path}/namespaces/${nsid}" > + > + if [[ $# -eq 4 ]]; then > + uuid="$4" > + fi > + > + mkdir "${ns_path}" > + printf "%s" "${blkdev}" > "${ns_path}/device_path" > + printf "%s" "${uuid}" > "${ns_path}/device_uuid" > + printf 1 > "${ns_path}/enable" > +} > + > +_create_nvmet_subsystem() { > + local nvmet_subsystem="$1" > + local blkdev="$2" > + local uuid=$3 > + local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > + > + mkdir -p "${cfs_path}" > + echo 0 > "${cfs_path}/attr_allow_any_host" > + _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" > +} > + > +_add_nvmet_allow_hosts() { > + local nvmet_subsystem="$1" > + local nvmet_hostnqn="$2" > + local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > + local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" > + > + ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" > +} > + > +_create_nvmet_host() { > + local nvmet_subsystem="$1" > + local nvmet_hostnqn="$2" > + local nvmet_hostkey="$3" > + local nvmet_ctrlkey="$4" > + local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" > + > + if [[ -d "${host_path}" ]]; then > + echo "FAIL target setup failed. stale host configuration found" > + return 1; > + fi > + > + mkdir "${host_path}" > + _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" > + if [[ "${nvmet_hostkey}" ]] ; then > + echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" > + fi > + if [[ "${nvmet_ctrlkey}" ]] ; then > + echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" > + fi > +} > + > +_remove_nvmet_ns() { > + local nvmet_subsystem="$1" > + local nsid=$2 > + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > + local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" > + > + echo 0 > "${nvmet_ns_path}/enable" > + rmdir "${nvmet_ns_path}" > +} > + > +_remove_nvmet_subsystem() { > + local nvmet_subsystem="$1" > + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > + > + _remove_nvmet_ns "${nvmet_subsystem}" "1" > + rm -f "${subsys_path}"/allowed_hosts/* > + rmdir "${subsys_path}" > +} > + > +_remove_nvmet_host() { > + local nvmet_host="$1" > + local host_path="${NVMET_CFS}/hosts/${nvmet_host}" > + > + rmdir "${host_path}" > +} > + > +_add_nvmet_subsys_to_port() { > + local port="$1" > + local nvmet_subsystem="$2" > + > + ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ > + "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" > +} > + > +_remove_nvmet_subsystem_from_port() { > + local port="$1" > + local nvmet_subsystem="$2" > + > + rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" > +} > + > +_get_nvmet_ports() { > + local nvmet_subsystem="$1" > + local -n nvmet_ports="$2" > + local cfs_path="${NVMET_CFS}/ports" > + local sarg > + > + sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" > + > + for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do > + nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") > + done > +} > + > +_find_nvme_dev() { > + local subsys=$1 > + local subsysnqn > + local dev > + for dev in /sys/class/nvme/nvme*; do > + [ -e "$dev" ] || continue > + dev="$(basename "$dev")" > + subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" > + if [[ "$subsysnqn" == "$subsys" ]]; then > + echo "$dev" > + fi > + done > +} > + > +_nvmet_target_cleanup() { > + local ports > + local port > + local blkdev > + local subsysnqn="${def_subsysnqn}" > + local blkdev_type="" > + > + while [[ $# -gt 0 ]]; do > + case $1 in > + --blkdev) > + blkdev_type="$2" > + shift 2 > + ;; > + --subsysnqn) > + subsysnqn="$2" > + shift 2 > + ;; > + *) > + echo "WARNING: unknown argument: $1" > + shift > + ;; > + esac > + done > + > + _get_nvmet_ports "${subsysnqn}" ports > + > + for port in "${ports[@]}"; do > + _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" > + _remove_nvmet_port "${port}" > + done > + _remove_nvmet_subsystem "${subsysnqn}" > + _remove_nvmet_host "${def_hostnqn}" > + > + if [[ "${blkdev_type}" == "device" ]]; then > + _cleanup_blkdev > + fi > +} > diff --git a/tests/nvme/rc b/tests/nvme/rc > index c1ddf41..3462f2e 100644 > --- a/tests/nvme/rc > +++ b/tests/nvme/rc > @@ -5,25 +5,9 @@ > # Test specific to NVMe devices > > . common/rc > +. common/nvme > . common/multipath-over-rdma > > -def_traddr="127.0.0.1" > -def_adrfam="ipv4" > -def_trsvcid="4420" > -def_remote_wwnn="0x10001100aa000001" > -def_remote_wwpn="0x20001100aa000001" > -def_local_wwnn="0x10001100aa000002" > -def_local_wwpn="0x20001100aa000002" > -def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" > -def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" > -export def_subsysnqn="blktests-subsystem-1" > -export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" > -_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" > -_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G > -_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 > -nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} > -NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} > - > _NVMET_TRTYPES_is_valid() { > local type > > @@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() { > COND_DESC="bd=${nvmet_blkdev_type}" > } > > -# TMPDIR can not be referred out of test() or test_device() context. Instead of > -# global variable def_flie_path, use this getter function. > -_nvme_def_file_path() { > - echo "${TMPDIR}/img" > -} > - > _nvme_requires() { > _have_program nvme > _require_nvme_test_img_size 4m > @@ -144,8 +122,6 @@ group_device_requires() { > _require_test_dev_is_nvme > } > > -NVMET_CFS="/sys/kernel/config/nvmet/" > - > _require_test_dev_is_nvme() { > if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then > SKIP_REASONS+=("$TEST_DEV is not a NVMe device") > @@ -168,31 +144,6 @@ _require_nvme_test_img_size() { > return 0 > } > > -_require_nvme_trtype() { > - local trtype > - for trtype in "$@"; do > - if [[ "${nvme_trtype}" == "$trtype" ]]; then > - return 0 > - fi > - done > - SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") > - return 1 > -} > - > -_require_nvme_trtype_is_loop() { > - if ! _require_nvme_trtype loop; then > - return 1 > - fi > - return 0 > -} > - > -_require_nvme_trtype_is_fabrics() { > - if ! _require_nvme_trtype loop fc rdma tcp; then > - return 1 > - fi > - return 0 > -} > - > _require_nvme_cli_auth() { > if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then > SKIP_REASONS+=("nvme gen-dhchap-key command missing") > @@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() { > echo "${io_size_kb}k" > } > > -_nvme_fcloop_add_rport() { > - local local_wwnn="$1" > - local local_wwpn="$2" > - local remote_wwnn="$3" > - local remote_wwpn="$4" > - local loopctl=/sys/class/fcloop/ctl > - > - echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port > -} > - > -_nvme_fcloop_add_lport() { > - local wwnn="$1" > - local wwpn="$2" > - local loopctl=/sys/class/fcloop/ctl > - > - echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port > -} > - > -_nvme_fcloop_add_tport() { > - local wwnn="$1" > - local wwpn="$2" > - local loopctl=/sys/class/fcloop/ctl > - > - echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port > -} > - > -_setup_fcloop() { > - local local_wwnn="${1:-$def_local_wwnn}" > - local local_wwpn="${2:-$def_local_wwpn}" > - local remote_wwnn="${3:-$def_remote_wwnn}" > - local remote_wwpn="${4:-$def_remote_wwpn}" > - > - _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" > - _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" > - _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ > - "${remote_wwnn}" "${remote_wwpn}" > -} > - > -_nvme_fcloop_del_rport() { > - local local_wwnn="$1" > - local local_wwpn="$2" > - local remote_wwnn="$3" > - local remote_wwpn="$4" > - local loopctl=/sys/class/fcloop/ctl > - > - if [[ ! -f "${loopctl}/del_remote_port" ]]; then > - return > - fi > - echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" > -} > - > -_nvme_fcloop_del_lport() { > - local wwnn="$1" > - local wwpn="$2" > - local loopctl=/sys/class/fcloop/ctl > - > - if [[ ! -f "${loopctl}/del_local_port" ]]; then > - return > - fi > - echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" > -} > - > -_nvme_fcloop_del_tport() { > - local wwnn="$1" > - local wwpn="$2" > - local loopctl=/sys/class/fcloop/ctl > - > - if [[ ! -f "${loopctl}/del_target_port" ]]; then > - return > - fi > - echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" > -} > - > -_cleanup_fcloop() { > - local local_wwnn="${1:-$def_local_wwnn}" > - local local_wwpn="${2:-$def_local_wwpn}" > - local remote_wwnn="${3:-$def_remote_wwnn}" > - local remote_wwpn="${4:-$def_remote_wwpn}" > - > - _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" > - _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" > - _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ > - "${remote_wwnn}" "${remote_wwpn}" > -} > - > -_cleanup_blkdev() { > - local blkdev > - local dev > - > - blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" > - for dev in ${blkdev}; do > - losetup -d "${dev}" > - done > - rm -f "$(_nvme_def_file_path)" > -} > - > -_cleanup_nvmet() { > - local dev > - local port > - local subsys > - local transport > - local name > - > - if [[ ! -d "${NVMET_CFS}" ]]; then > - return 0 > - fi > - > - # Don't let successive Ctrl-Cs interrupt the cleanup processes > - trap '' SIGINT > - > - shopt -s nullglob > - > - for dev in /sys/class/nvme/nvme*; do > - dev="$(basename "$dev")" > - transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" > - if [[ "$transport" == "${nvme_trtype}" ]]; then > - # if udev auto connect is enabled for FC we get false positives > - if [[ "$transport" != "fc" ]]; then > - echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" > - fi > - _nvme_disconnect_ctrl "${dev}" 2>/dev/null > - fi > - done > - > - for port in "${NVMET_CFS}"/ports/*; do > - name=$(basename "${port}") > - echo "WARNING: Test did not clean up port: ${name}" > - rm -f "${port}"/subsystems/* > - rmdir "${port}" > - done > - > - for subsys in "${NVMET_CFS}"/subsystems/*; do > - name=$(basename "${subsys}") > - echo "WARNING: Test did not clean up subsystem: ${name}" > - for ns in "${subsys}"/namespaces/*; do > - rmdir "${ns}" > - done > - rmdir "${subsys}" > - done > - > - for host in "${NVMET_CFS}"/hosts/*; do > - name=$(basename "${host}") > - echo "WARNING: Test did not clean up host: ${name}" > - rmdir "${host}" > - done > - > - shopt -u nullglob > - trap SIGINT > - > - if [[ "${nvme_trtype}" == "fc" ]]; then > - _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ > - "${def_remote_wwnn}" "${def_remote_wwpn}" > - modprobe -rq nvme-fcloop 2>/dev/null > - fi > - modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null > - if [[ "${nvme_trtype}" != "loop" ]]; then > - modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null > - fi > - modprobe -rq nvmet 2>/dev/null > - if [[ "${nvme_trtype}" == "rdma" ]]; then > - stop_soft_rdma > - fi > - > - _cleanup_blkdev > -} > - > -_setup_nvmet() { > - _register_test_cleanup _cleanup_nvmet > - modprobe -q nvmet > - if [[ "${nvme_trtype}" != "loop" ]]; then > - modprobe -q nvmet-"${nvme_trtype}" > - fi > - modprobe -q nvme-"${nvme_trtype}" > - if [[ "${nvme_trtype}" == "rdma" ]]; then > - start_soft_rdma > - for i in $(rdma_network_interfaces) > - do > - if [[ "${nvme_adrfam}" == "ipv6" ]]; then > - ipv6_addr=$(get_ipv6_ll_addr "$i") > - if [[ -n "${ipv6_addr}" ]]; then > - def_traddr=${ipv6_addr} > - fi > - else > - ipv4_addr=$(get_ipv4_addr "$i") > - if [[ -n "${ipv4_addr}" ]]; then > - def_traddr=${ipv4_addr} > - fi > - fi > - done > - fi > - if [[ "${nvme_trtype}" = "fc" ]]; then > - modprobe -q nvme-fcloop > - _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ > - "${def_remote_wwnn}" "${def_remote_wwpn}" > - > - def_traddr=$(printf "nn-%s:pn-%s" \ > - "${def_remote_wwnn}" \ > - "${def_remote_wwpn}") > - def_host_traddr=$(printf "nn-%s:pn-%s" \ > - "${def_local_wwnn}" \ > - "${def_local_wwpn}") > - fi > -} > - > -_nvme_disconnect_ctrl() { > - local ctrl="$1" > - > - nvme disconnect --device "${ctrl}" > -} > - > _nvme_disconnect_subsys() { > local subsysnqn="$def_subsysnqn" > > @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() { > grep -o "disconnected.*" > } > > -_nvme_connect_subsys() { > - local subsysnqn="$def_subsysnqn" > - local hostnqn="$def_hostnqn" It looks weird that _nvme_connect_subsys() is moved to common/nvme, but _nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to move _nvme_disconnect_subsys() also. Currently, md/001 does not use _nvme_disconnect_subsys(). Isn't it the better to call it in cleanup_nvme_over_tcp()? > - local hostid="$def_hostid" > - local hostkey="" > - local ctrlkey="" > - local nr_io_queues="" > - local nr_write_queues="" > - local nr_poll_queues="" > - local keep_alive_tmo="" > - local reconnect_delay="" > - local ctrl_loss_tmo="" > - local no_wait=false > - local i > - > - while [[ $# -gt 0 ]]; do > - case $1 in > - --subsysnqn) > - subsysnqn="$2" > - shift 2 > - ;; > - --hostnqn) > - hostnqn="$2" > - shift 2 > - ;; > - --hostid) > - hostid="$2" > - shift 2 > - ;; > - --dhchap-secret) > - hostkey="$2" > - shift 2 > - ;; > - --dhchap-ctrl-secret) > - ctrlkey="$2" > - shift 2 > - ;; > - --nr-io-queues) > - nr_io_queues="$2" > - shift 2 > - ;; > - --nr-write-queues) > - nr_write_queues="$2" > - shift 2 > - ;; > - --nr-poll-queues) > - nr_poll_queues="$2" > - shift 2 > - ;; > - --keep-alive-tmo) > - keep_alive_tmo="$2" > - shift 2 > - ;; > - --reconnect-delay) > - reconnect_delay="$2" > - shift 2 > - ;; > - --ctrl-loss-tmo) > - ctrl_loss_tmo="$2" > - shift 2 > - ;; > - --no-wait) > - no_wait=true > - shift 1 > - ;; > - *) > - echo "WARNING: unknown argument: $1" > - shift > - ;; > - esac > - done > - > - ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") > - if [[ "${nvme_trtype}" == "fc" ]] ; then > - ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") > - elif [[ "${nvme_trtype}" != "loop" ]]; then > - ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") > - fi > - ARGS+=(--hostnqn="${hostnqn}") > - ARGS+=(--hostid="${hostid}") > - if [[ -n "${hostkey}" ]]; then > - ARGS+=(--dhchap-secret="${hostkey}") > - fi > - if [[ -n "${ctrlkey}" ]]; then > - ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") > - fi > - if [[ -n "${nr_io_queues}" ]]; then > - ARGS+=(--nr-io-queues="${nr_io_queues}") > - fi > - if [[ -n "${nr_write_queues}" ]]; then > - ARGS+=(--nr-write-queues="${nr_write_queues}") > - fi > - if [[ -n "${nr_poll_queues}" ]]; then > - ARGS+=(--nr-poll-queues="${nr_poll_queues}") > - fi > - if [[ -n "${keep_alive_tmo}" ]]; then > - ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") > - fi > - if [[ -n "${reconnect_delay}" ]]; then > - ARGS+=(--reconnect-delay="${reconnect_delay}") > - fi > - if [[ -n "${ctrl_loss_tmo}" ]]; then > - ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") > - fi > - > - nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" > - > - # Wait until device file and uuid/wwid sysfs attributes get ready for > - # all namespaces. > - if [[ ${no_wait} = false ]]; then > - udevadm settle > - for ((i = 0; i < 10; i++)); do > - _nvme_ns_ready "${subsysnqn}" && return > - sleep .1 > - done > - fi > -} > - > -_nvme_ns_ready() { > - local subsysnqn="${1}" > - local ns_path ns_id dev > - local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" > - > - dev=$(_find_nvme_dev "$subsysnqn") > - for ns_path in "${cfs_path}/namespaces/"*; do > - ns_id=${ns_path##*/} > - if [[ ! -b /dev/${dev}n${ns_id} || > - ! -e /sys/block/${dev}n${ns_id}/uuid || > - ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then > - return 1 > - fi > - done > - return 0 > -} > - > _nvme_discover() { > local trtype="$1" > local traddr="${2:-$def_traddr}" > @@ -617,73 +223,6 @@ _nvme_discover() { > nvme discover "${ARGS[@]}" > } > > -_create_nvmet_port() { > - local trtype="$1" > - local traddr="${2:-$def_traddr}" > - local adrfam="${3:-$def_adrfam}" > - local trsvcid="${4:-$def_trsvcid}" > - > - local port > - for ((port = 0; ; port++)); do > - if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then > - break > - fi > - done > - > - mkdir "${NVMET_CFS}/ports/${port}" > - echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" > - echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" > - echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" > - if [[ "${adrfam}" != "fc" ]]; then > - echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" > - fi > - > - echo "${port}" > -} > - > -_remove_nvmet_port() { > - local port="$1" > - rmdir "${NVMET_CFS}/ports/${port}" > -} > - > -_create_nvmet_ns() { > - local nvmet_subsystem="$1" > - local nsid="$2" > - local blkdev="$3" > - local uuid="00000000-0000-0000-0000-000000000000" > - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > - local ns_path="${subsys_path}/namespaces/${nsid}" > - > - if [[ $# -eq 4 ]]; then > - uuid="$4" > - fi > - > - mkdir "${ns_path}" > - printf "%s" "${blkdev}" > "${ns_path}/device_path" > - printf "%s" "${uuid}" > "${ns_path}/device_uuid" > - printf 1 > "${ns_path}/enable" > -} > - > -_create_nvmet_subsystem() { > - local nvmet_subsystem="$1" > - local blkdev="$2" > - local uuid=$3 > - local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > - > - mkdir -p "${cfs_path}" > - echo 0 > "${cfs_path}/attr_allow_any_host" > - _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" > -} > - > -_add_nvmet_allow_hosts() { > - local nvmet_subsystem="$1" > - local nvmet_hostnqn="$2" > - local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > - local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" > - > - ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" > -} > - > _remove_nvmet_allow_hosts() { > local nvmet_subsystem="$1" > local nvmet_hostnqn="$2" > @@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() { > rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" > } > > -_create_nvmet_host() { > - local nvmet_subsystem="$1" > - local nvmet_hostnqn="$2" > - local nvmet_hostkey="$3" > - local nvmet_ctrlkey="$4" > - local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" > - > - if [[ -d "${host_path}" ]]; then > - echo "FAIL target setup failed. stale host configuration found" > - return 1; > - fi > - > - mkdir "${host_path}" > - _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" > - if [[ "${nvmet_hostkey}" ]] ; then > - echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" > - fi > - if [[ "${nvmet_ctrlkey}" ]] ; then > - echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" > - fi > -} > - > -_remove_nvmet_ns() { > - local nvmet_subsystem="$1" > - local nsid=$2 > - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > - local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" > - > - echo 0 > "${nvmet_ns_path}/enable" > - rmdir "${nvmet_ns_path}" > -} > - > -_remove_nvmet_subsystem() { > - local nvmet_subsystem="$1" > - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > - > - _remove_nvmet_ns "${nvmet_subsystem}" "1" > - rm -f "${subsys_path}"/allowed_hosts/* > - rmdir "${subsys_path}" > -} > - > -_remove_nvmet_host() { > - local nvmet_host="$1" > - local host_path="${NVMET_CFS}/hosts/${nvmet_host}" > - > - rmdir "${host_path}" > -} > - > _create_nvmet_passthru() { > local nvmet_subsystem="$1" > local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" > @@ -765,34 +256,6 @@ _remove_nvmet_passhtru() { > rmdir "${subsys_path}" > } > > -_add_nvmet_subsys_to_port() { > - local port="$1" > - local nvmet_subsystem="$2" > - > - ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ > - "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" > -} > - > -_remove_nvmet_subsystem_from_port() { > - local port="$1" > - local nvmet_subsystem="$2" > - > - rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" > -} > - > -_get_nvmet_ports() { > - local nvmet_subsystem="$1" > - local -n nvmet_ports="$2" > - local cfs_path="${NVMET_CFS}/ports" > - local sarg > - > - sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" > - > - for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do > - nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") > - done > -} > - > _set_nvmet_hostkey() { > local nvmet_hostnqn="$1" > local nvmet_hostkey="$2" > @@ -829,20 +292,6 @@ _set_nvmet_dhgroup() { > "${cfs_path}/dhchap_dhgroup" > } > > -_find_nvme_dev() { > - local subsys=$1 > - local subsysnqn > - local dev > - for dev in /sys/class/nvme/nvme*; do > - [ -e "$dev" ] || continue > - dev="$(basename "$dev")" > - subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" > - if [[ "$subsysnqn" == "$subsys" ]]; then > - echo "$dev" > - fi > - done > -} > - > _find_nvme_ns() { > local subsys_uuid=$1 > local uuid > @@ -924,44 +373,6 @@ _nvmet_target_setup() { > "${hostkey}" "${ctrlkey}" > } > > -_nvmet_target_cleanup() { > - local ports > - local port > - local blkdev > - local subsysnqn="${def_subsysnqn}" > - local blkdev_type="" > - > - while [[ $# -gt 0 ]]; do > - case $1 in > - --blkdev) > - blkdev_type="$2" > - shift 2 > - ;; > - --subsysnqn) > - subsysnqn="$2" > - shift 2 > - ;; > - *) > - echo "WARNING: unknown argument: $1" > - shift > - ;; > - esac > - done > - > - _get_nvmet_ports "${subsysnqn}" ports > - > - for port in "${ports[@]}"; do > - _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" > - _remove_nvmet_port "${port}" > - done > - _remove_nvmet_subsystem "${subsysnqn}" > - _remove_nvmet_host "${def_hostnqn}" > - > - if [[ "${blkdev_type}" == "device" ]]; then > - _cleanup_blkdev > - fi > -} > - > _nvmet_passthru_target_setup() { > local subsysnqn="$def_subsysnqn" > local port > -- > 2.45.1 >
On 6/26/24 14:18, Shinichiro Kawasaki wrote: > On Jun 24, 2024 / 13:46, Ofir Gal wrote: >> Move functions from tests/nvme/rc to common/nvme to be able to reuse >> them in other tests groups. >> >> Signed-off-by: Ofir Gal <ofir.gal@volumez.com> >> --- >> I have moved the function that are nessecary for the regression test I >> add. Let me know if we want to move more or all the functions to >> common/nvme. >> >> shellcheck detects 2 new warnings: >> common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154] >> common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154] >> >> I have tried to figure out why but I couldn't figure out what is >> different than the upstream version. How can I fix them? > > Hi Ofir, thanks for this work. Please find two comments in line below. One of > them suggests a fix for the shellcheck warnings. > Applied to v3 >> >> >> common/nvme  | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++ >> tests/nvme/rc | 591 +------------------------------------------------ >> 2 files changed, 596 insertions(+), 590 deletions(-) >> create mode 100644 common/nvme >> >> diff --git a/common/nvme b/common/nvme >> new file mode 100644 >> index 0000000..1800263 >> --- /dev/null >> +++ b/common/nvme >> @@ -0,0 +1,595 @@ >> +#!/bin/bash >> +# SPDX-License-Identifier: GPL-3.0+ >> +# >> +# nvme helper functions. >> + >> +. common/shellcheck >> + >> +def_traddr="127.0.0.1" >> +def_adrfam="ipv4" >> +def_trsvcid="4420" >> +def_remote_wwnn="0x10001100aa000001" >> +def_remote_wwpn="0x20001100aa000001" >> +def_local_wwnn="0x10001100aa000002" >> +def_local_wwpn="0x20001100aa000002" >> +def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" >> +def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" >> +export def_subsysnqn="blktests-subsystem-1" >> +export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" >> +_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" >> +_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G >> +_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 >> +nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} >> +NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} >> +NVMET_CFS="/sys/kernel/config/nvmet/" > > As for the shellcheck warnings, I suggestto add two lines below here to suppress > them. > > nvme_trtype=${nvme_trtype:-} > nvme_adrfam=${nvme_adrfam:-} > Thanks, applied to v3 >> + >> +# TMPDIR can not be referred out of test() or test_device() context. Instead of >> +# global variable def_flie_path, use this getter function. >> +_nvme_def_file_path() { >> +   echo "${TMPDIR}/img" >> +} >> + >> +_require_nvme_trtype() { >> +   local trtype >> +   for trtype in "$@"; do >> +      if [[ "${nvme_trtype}" == "$trtype" ]]; then >> +         return 0 >> +      fi >> +   done >> +   SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") >> +   return 1 >> +} >> + >> +_require_nvme_trtype_is_loop() { >> +   if ! _require_nvme_trtype loop; then >> +      return 1 >> +   fi >> +   return 0 >> +} >> + >> +_require_nvme_trtype_is_fabrics() { >> +   if ! _require_nvme_trtype loop fc rdma tcp; then >> +      return 1 >> +   fi >> +   return 0 >> +} >> + >> +_nvme_fcloop_add_rport() { >> +   local local_wwnn="$1" >> +   local local_wwpn="$2" >> +   local remote_wwnn="$3" >> +   local remote_wwpn="$4" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port >> +} >> + >> +_nvme_fcloop_add_lport() { >> +   local wwnn="$1" >> +   local wwpn="$2" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port >> +} >> + >> +_nvme_fcloop_add_tport() { >> +   local wwnn="$1" >> +   local wwpn="$2" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port >> +} >> + >> +_setup_fcloop() { >> +   local local_wwnn="${1:-$def_local_wwnn}" >> +   local local_wwpn="${2:-$def_local_wwpn}" >> +   local remote_wwnn="${3:-$def_remote_wwnn}" >> +   local remote_wwpn="${4:-$def_remote_wwpn}" >> + >> +   _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" >> +   _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" >> +   _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ >> +                    "${remote_wwnn}" "${remote_wwpn}" >> +} >> + >> +_nvme_fcloop_del_rport() { >> +   local local_wwnn="$1" >> +   local local_wwpn="$2" >> +   local remote_wwnn="$3" >> +   local remote_wwpn="$4" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   if [[ ! -f "${loopctl}/del_remote_port" ]]; then >> +      return >> +   fi >> +   echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" >> +} >> + >> +_nvme_fcloop_del_lport() { >> +   local wwnn="$1" >> +   local wwpn="$2" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   if [[ ! -f "${loopctl}/del_local_port" ]]; then >> +      return >> +   fi >> +   echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" >> +} >> + >> +_nvme_fcloop_del_tport() { >> +   local wwnn="$1" >> +   local wwpn="$2" >> +   local loopctl=/sys/class/fcloop/ctl >> + >> +   if [[ ! -f "${loopctl}/del_target_port" ]]; then >> +      return >> +   fi >> +   echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" >> +} >> + >> +_cleanup_fcloop() { >> +   local local_wwnn="${1:-$def_local_wwnn}" >> +   local local_wwpn="${2:-$def_local_wwpn}" >> +   local remote_wwnn="${3:-$def_remote_wwnn}" >> +   local remote_wwpn="${4:-$def_remote_wwpn}" >> + >> +   _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" >> +   _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" >> +   _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ >> +               "${remote_wwnn}" "${remote_wwpn}" >> +} >> + >> +_cleanup_blkdev() { >> +   local blkdev >> +   local dev >> + >> +   blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" >> +   for dev in ${blkdev}; do >> +      losetup -d "${dev}" >> +   done >> +   rm -f "$(_nvme_def_file_path)" >> +} >> + >> +_cleanup_nvmet() { >> +   local dev >> +   local port >> +   local subsys >> +   local transport >> +   local name >> + >> +   if [[ ! -d "${NVMET_CFS}" ]]; then >> +      return 0 >> +   fi >> + >> +   # Don't let successive Ctrl-Cs interrupt the cleanup processes >> +   trap '' SIGINT >> + >> +   shopt -s nullglob >> + >> +   for dev in /sys/class/nvme/nvme*; do >> +      dev="$(basename "$dev")" >> +      transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" >> +      if [[ "$transport" == "${nvme_trtype}" ]]; then >> +         # if udev auto connect is enabled for FC we get false positives >> +         if [[ "$transport" != "fc" ]]; then >> +            echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" >> +         fi >> +         _nvme_disconnect_ctrl "${dev}" 2>/dev/null >> +      fi >> +   done >> + >> +   for port in "${NVMET_CFS}"/ports/*; do >> +      name=$(basename "${port}") >> +      echo "WARNING: Test did not clean up port: ${name}" >> +      rm -f "${port}"/subsystems/* >> +      rmdir "${port}" >> +   done >> + >> +   for subsys in "${NVMET_CFS}"/subsystems/*; do >> +      name=$(basename "${subsys}") >> +      echo "WARNING: Test did not clean up subsystem: ${name}" >> +      for ns in "${subsys}"/namespaces/*; do >> +         rmdir "${ns}" >> +      done >> +      rmdir "${subsys}" >> +   done >> + >> +   for host in "${NVMET_CFS}"/hosts/*; do >> +      name=$(basename "${host}") >> +      echo "WARNING: Test did not clean up host: ${name}" >> +      rmdir "${host}" >> +   done >> + >> +   shopt -u nullglob >> +   trap SIGINT >> + >> +   if [[ "${nvme_trtype}" == "fc" ]]; then >> +      _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ >> +            "${def_remote_wwnn}" "${def_remote_wwpn}" >> +      modprobe -rq nvme-fcloop 2>/dev/null >> +   fi >> +   modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null >> +   if [[ "${nvme_trtype}" != "loop" ]]; then >> +      modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null >> +   fi >> +   modprobe -rq nvmet 2>/dev/null >> +   if [[ "${nvme_trtype}" == "rdma" ]]; then >> +      stop_soft_rdma >> +   fi >> + >> +   _cleanup_blkdev >> +} >> + >> +_setup_nvmet() { >> +   _register_test_cleanup _cleanup_nvmet >> +   modprobe -q nvmet >> +   if [[ "${nvme_trtype}" != "loop" ]]; then >> +      modprobe -q nvmet-"${nvme_trtype}" >> +   fi >> +   modprobe -q nvme-"${nvme_trtype}" >> +   if [[ "${nvme_trtype}" == "rdma" ]]; then >> +      start_soft_rdma >> +      for i in $(rdma_network_interfaces) >> +      do >> +         if [[ "${nvme_adrfam}" == "ipv6" ]]; then >> +            ipv6_addr=$(get_ipv6_ll_addr "$i") >> +            if [[ -n "${ipv6_addr}" ]]; then >> +               def_traddr=${ipv6_addr} >> +            fi >> +         else >> +            ipv4_addr=$(get_ipv4_addr "$i") >> +            if [[ -n "${ipv4_addr}" ]]; then >> +               def_traddr=${ipv4_addr} >> +            fi >> +         fi >> +      done >> +   fi >> +   if [[ "${nvme_trtype}" = "fc" ]]; then >> +      modprobe -q nvme-fcloop >> +      _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ >> +              "${def_remote_wwnn}" "${def_remote_wwpn}" >> + >> +      def_traddr=$(printf "nn-%s:pn-%s" \ >> +               "${def_remote_wwnn}" \ >> +               "${def_remote_wwpn}") >> +      def_host_traddr=$(printf "nn-%s:pn-%s" \ >> +               "${def_local_wwnn}" \ >> +               "${def_local_wwpn}") >> +   fi >> +} >> + >> +_nvme_disconnect_ctrl() { >> +   local ctrl="$1" >> + >> +   nvme disconnect --device "${ctrl}" >> +} >> + >> +_nvme_connect_subsys() { >> +   local subsysnqn="$def_subsysnqn" >> +   local hostnqn="$def_hostnqn" >> +   local hostid="$def_hostid" >> +   local hostkey="" >> +   local ctrlkey="" >> +   local nr_io_queues="" >> +   local nr_write_queues="" >> +   local nr_poll_queues="" >> +   local keep_alive_tmo="" >> +   local reconnect_delay="" >> +   local ctrl_loss_tmo="" >> +   local no_wait=false >> +   local i >> + >> +   while [[ $# -gt 0 ]]; do >> +      case $1 in >> +         --subsysnqn) >> +            subsysnqn="$2" >> +            shift 2 >> +            ;; >> +         --hostnqn) >> +            hostnqn="$2" >> +            shift 2 >> +            ;; >> +         --hostid) >> +            hostid="$2" >> +            shift 2 >> +            ;; >> +         --dhchap-secret) >> +            hostkey="$2" >> +            shift 2 >> +            ;; >> +         --dhchap-ctrl-secret) >> +            ctrlkey="$2" >> +            shift 2 >> +            ;; >> +         --nr-io-queues) >> +            nr_io_queues="$2" >> +            shift 2 >> +            ;; >> +         --nr-write-queues) >> +            nr_write_queues="$2" >> +            shift 2 >> +            ;; >> +         --nr-poll-queues) >> +            nr_poll_queues="$2" >> +            shift 2 >> +            ;; >> +         --keep-alive-tmo) >> +            keep_alive_tmo="$2" >> +            shift 2 >> +            ;; >> +         --reconnect-delay) >> +            reconnect_delay="$2" >> +            shift 2 >> +            ;; >> +         --ctrl-loss-tmo) >> +            ctrl_loss_tmo="$2" >> +            shift 2 >> +            ;; >> +         --no-wait) >> +            no_wait=true >> +            shift 1 >> +            ;; >> +         *) >> +            echo "WARNING: unknown argument: $1" >> +            shift >> +            ;; >> +      esac >> +   done >> + >> +   ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") >> +   if [[ "${nvme_trtype}" == "fc" ]] ; then >> +      ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") >> +   elif [[ "${nvme_trtype}" != "loop" ]]; then >> +      ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") >> +   fi >> +   ARGS+=(--hostnqn="${hostnqn}") >> +   ARGS+=(--hostid="${hostid}") >> +   if [[ -n "${hostkey}" ]]; then >> +      ARGS+=(--dhchap-secret="${hostkey}") >> +   fi >> +   if [[ -n "${ctrlkey}" ]]; then >> +      ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") >> +   fi >> +   if [[ -n "${nr_io_queues}" ]]; then >> +      ARGS+=(--nr-io-queues="${nr_io_queues}") >> +   fi >> +   if [[ -n "${nr_write_queues}" ]]; then >> +      ARGS+=(--nr-write-queues="${nr_write_queues}") >> +   fi >> +   if [[ -n "${nr_poll_queues}" ]]; then >> +      ARGS+=(--nr-poll-queues="${nr_poll_queues}") >> +   fi >> +   if [[ -n "${keep_alive_tmo}" ]]; then >> +      ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") >> +   fi >> +   if [[ -n "${reconnect_delay}" ]]; then >> +      ARGS+=(--reconnect-delay="${reconnect_delay}") >> +   fi >> +   if [[ -n "${ctrl_loss_tmo}" ]]; then >> +      ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") >> +   fi >> + >> +   nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" >> + >> +   # Wait until device file and uuid/wwid sysfs attributes get ready for >> +   # all namespaces. >> +   if [[ ${no_wait} = false ]]; then >> +      udevadm settle >> +      for ((i = 0; i < 10; i++)); do >> +         _nvme_ns_ready "${subsysnqn}" && return >> +         sleep .1 >> +      done >> +   fi >> +} >> + >> +_nvme_ns_ready() { >> +   local subsysnqn="${1}" >> +   local ns_path ns_id dev >> +   local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" >> + >> +   dev=$(_find_nvme_dev "$subsysnqn") >> +   for ns_path in "${cfs_path}/namespaces/"*; do >> +      ns_id=${ns_path##*/} >> +      if [[ ! -b /dev/${dev}n${ns_id} || >> +           ! -e /sys/block/${dev}n${ns_id}/uuid || >> +           ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then >> +         return 1 >> +      fi >> +   done >> +   return 0 >> +} >> + >> +_create_nvmet_port() { >> +   local trtype="$1" >> +   local traddr="${2:-$def_traddr}" >> +   local adrfam="${3:-$def_adrfam}" >> +   local trsvcid="${4:-$def_trsvcid}" >> + >> +   local port >> +   for ((port = 0; ; port++)); do >> +      if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then >> +         break >> +      fi >> +   done >> + >> +   mkdir "${NVMET_CFS}/ports/${port}" >> +   echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" >> +   echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" >> +   echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" >> +   if [[ "${adrfam}" != "fc" ]]; then >> +      echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" >> +   fi >> + >> +   echo "${port}" >> +} >> + >> +_remove_nvmet_port() { >> +   local port="$1" >> +   rmdir "${NVMET_CFS}/ports/${port}" >> +} >> + >> +_create_nvmet_ns() { >> +   local nvmet_subsystem="$1" >> +   local nsid="$2" >> +   local blkdev="$3" >> +   local uuid="00000000-0000-0000-0000-000000000000" >> +   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> +   local ns_path="${subsys_path}/namespaces/${nsid}" >> + >> +   if [[ $# -eq 4 ]]; then >> +      uuid="$4" >> +   fi >> + >> +   mkdir "${ns_path}" >> +   printf "%s" "${blkdev}" > "${ns_path}/device_path" >> +   printf "%s" "${uuid}" > "${ns_path}/device_uuid" >> +   printf 1 > "${ns_path}/enable" >> +} >> + >> +_create_nvmet_subsystem() { >> +   local nvmet_subsystem="$1" >> +   local blkdev="$2" >> +   local uuid=$3 >> +   local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> + >> +   mkdir -p "${cfs_path}" >> +   echo 0 > "${cfs_path}/attr_allow_any_host" >> +   _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" >> +} >> + >> +_add_nvmet_allow_hosts() { >> +   local nvmet_subsystem="$1" >> +   local nvmet_hostnqn="$2" >> +   local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> +   local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" >> + >> +   ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" >> +} >> + >> +_create_nvmet_host() { >> +   local nvmet_subsystem="$1" >> +   local nvmet_hostnqn="$2" >> +   local nvmet_hostkey="$3" >> +   local nvmet_ctrlkey="$4" >> +   local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" >> + >> +   if [[ -d "${host_path}" ]]; then >> +      echo "FAIL target setup failed. stale host configuration found" >> +      return 1; >> +   fi >> + >> +   mkdir "${host_path}" >> +   _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" >> +   if [[ "${nvmet_hostkey}" ]] ; then >> +      echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" >> +   fi >> +   if [[ "${nvmet_ctrlkey}" ]] ; then >> +      echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" >> +   fi >> +} >> + >> +_remove_nvmet_ns() { >> +   local nvmet_subsystem="$1" >> +   local nsid=$2 >> +   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> +   local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" >> + >> +   echo 0 > "${nvmet_ns_path}/enable" >> +   rmdir "${nvmet_ns_path}" >> +} >> + >> +_remove_nvmet_subsystem() { >> +   local nvmet_subsystem="$1" >> +   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> + >> +   _remove_nvmet_ns "${nvmet_subsystem}" "1" >> +   rm -f "${subsys_path}"/allowed_hosts/* >> +   rmdir "${subsys_path}" >> +} >> + >> +_remove_nvmet_host() { >> +   local nvmet_host="$1" >> +   local host_path="${NVMET_CFS}/hosts/${nvmet_host}" >> + >> +   rmdir "${host_path}" >> +} >> + >> +_add_nvmet_subsys_to_port() { >> +   local port="$1" >> +   local nvmet_subsystem="$2" >> + >> +   ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ >> +      "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" >> +} >> + >> +_remove_nvmet_subsystem_from_port() { >> +   local port="$1" >> +   local nvmet_subsystem="$2" >> + >> +   rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" >> +} >> + >> +_get_nvmet_ports() { >> +   local nvmet_subsystem="$1" >> +   local -n nvmet_ports="$2" >> +   local cfs_path="${NVMET_CFS}/ports" >> +   local sarg >> + >> +   sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" >> + >> +   for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do >> +      nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") >> +   done >> +} >> + >> +_find_nvme_dev() { >> +   local subsys=$1 >> +   local subsysnqn >> +   local dev >> +   for dev in /sys/class/nvme/nvme*; do >> +      [ -e "$dev" ] || continue >> +      dev="$(basename "$dev")" >> +      subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" >> +      if [[ "$subsysnqn" == "$subsys" ]]; then >> +         echo "$dev" >> +      fi >> +   done >> +} >> + >> +_nvmet_target_cleanup() { >> +   local ports >> +   local port >> +   local blkdev >> +   local subsysnqn="${def_subsysnqn}" >> +   local blkdev_type="" >> + >> +   while [[ $# -gt 0 ]]; do >> +      case $1 in >> +         --blkdev) >> +            blkdev_type="$2" >> +            shift 2 >> +            ;; >> +         --subsysnqn) >> +            subsysnqn="$2" >> +            shift 2 >> +            ;; >> +         *) >> +            echo "WARNING: unknown argument: $1" >> +            shift >> +            ;; >> +      esac >> +   done >> + >> +   _get_nvmet_ports "${subsysnqn}" ports >> + >> +   for port in "${ports[@]}"; do >> +      _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" >> +      _remove_nvmet_port "${port}" >> +   done >> +   _remove_nvmet_subsystem "${subsysnqn}" >> +   _remove_nvmet_host "${def_hostnqn}" >> + >> +   if [[ "${blkdev_type}" == "device" ]]; then >> +      _cleanup_blkdev >> +   fi >> +} >> diff --git a/tests/nvme/rc b/tests/nvme/rc >> index c1ddf41..3462f2e 100644 >> --- a/tests/nvme/rc >> +++ b/tests/nvme/rc >> @@ -5,25 +5,9 @@ >> # Test specific to NVMe devices >>  >> . common/rc >> +. common/nvme >> . common/multipath-over-rdma >>  >> -def_traddr="127.0.0.1" >> -def_adrfam="ipv4" >> -def_trsvcid="4420" >> -def_remote_wwnn="0x10001100aa000001" >> -def_remote_wwpn="0x20001100aa000001" >> -def_local_wwnn="0x10001100aa000002" >> -def_local_wwpn="0x20001100aa000002" >> -def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" >> -def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" >> -export def_subsysnqn="blktests-subsystem-1" >> -export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" >> -_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" >> -_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G >> -_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 >> -nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} >> -NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} >> - >> _NVMET_TRTYPES_is_valid() { >>     local type >>  >> @@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() { >>     COND_DESC="bd=${nvmet_blkdev_type}" >> } >>  >> -# TMPDIR can not be referred out of test() or test_device() context. Instead of >> -# global variable def_flie_path, use this getter function. >> -_nvme_def_file_path() { >> -   echo "${TMPDIR}/img" >> -} >> - >> _nvme_requires() { >>     _have_program nvme >>     _require_nvme_test_img_size 4m >> @@ -144,8 +122,6 @@ group_device_requires() { >>     _require_test_dev_is_nvme >> } >>  >> -NVMET_CFS="/sys/kernel/config/nvmet/" >> - >> _require_test_dev_is_nvme() { >>     if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then >>        SKIP_REASONS+=("$TEST_DEV is not a NVMe device") >> @@ -168,31 +144,6 @@ _require_nvme_test_img_size() { >>     return 0 >> } >>  >> -_require_nvme_trtype() { >> -   local trtype >> -   for trtype in "$@"; do >> -      if [[ "${nvme_trtype}" == "$trtype" ]]; then >> -         return 0 >> -      fi >> -   done >> -   SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") >> -   return 1 >> -} >> - >> -_require_nvme_trtype_is_loop() { >> -   if ! _require_nvme_trtype loop; then >> -      return 1 >> -   fi >> -   return 0 >> -} >> - >> -_require_nvme_trtype_is_fabrics() { >> -   if ! _require_nvme_trtype loop fc rdma tcp; then >> -      return 1 >> -   fi >> -   return 0 >> -} >> - >> _require_nvme_cli_auth() { >>     if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then >>        SKIP_REASONS+=("nvme gen-dhchap-key command missing") >> @@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() { >>     echo "${io_size_kb}k" >> } >>  >> -_nvme_fcloop_add_rport() { >> -   local local_wwnn="$1" >> -   local local_wwpn="$2" >> -   local remote_wwnn="$3" >> -   local remote_wwpn="$4" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port >> -} >> - >> -_nvme_fcloop_add_lport() { >> -   local wwnn="$1" >> -   local wwpn="$2" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port >> -} >> - >> -_nvme_fcloop_add_tport() { >> -   local wwnn="$1" >> -   local wwpn="$2" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port >> -} >> - >> -_setup_fcloop() { >> -   local local_wwnn="${1:-$def_local_wwnn}" >> -   local local_wwpn="${2:-$def_local_wwpn}" >> -   local remote_wwnn="${3:-$def_remote_wwnn}" >> -   local remote_wwpn="${4:-$def_remote_wwpn}" >> - >> -   _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" >> -   _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" >> -   _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ >> -                    "${remote_wwnn}" "${remote_wwpn}" >> -} >> - >> -_nvme_fcloop_del_rport() { >> -   local local_wwnn="$1" >> -   local local_wwpn="$2" >> -   local remote_wwnn="$3" >> -   local remote_wwpn="$4" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   if [[ ! -f "${loopctl}/del_remote_port" ]]; then >> -      return >> -   fi >> -   echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" >> -} >> - >> -_nvme_fcloop_del_lport() { >> -   local wwnn="$1" >> -   local wwpn="$2" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   if [[ ! -f "${loopctl}/del_local_port" ]]; then >> -      return >> -   fi >> -   echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" >> -} >> - >> -_nvme_fcloop_del_tport() { >> -   local wwnn="$1" >> -   local wwpn="$2" >> -   local loopctl=/sys/class/fcloop/ctl >> - >> -   if [[ ! -f "${loopctl}/del_target_port" ]]; then >> -      return >> -   fi >> -   echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" >> -} >> - >> -_cleanup_fcloop() { >> -   local local_wwnn="${1:-$def_local_wwnn}" >> -   local local_wwpn="${2:-$def_local_wwpn}" >> -   local remote_wwnn="${3:-$def_remote_wwnn}" >> -   local remote_wwpn="${4:-$def_remote_wwpn}" >> - >> -   _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" >> -   _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" >> -   _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ >> -               "${remote_wwnn}" "${remote_wwpn}" >> -} >> - >> -_cleanup_blkdev() { >> -   local blkdev >> -   local dev >> - >> -   blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" >> -   for dev in ${blkdev}; do >> -      losetup -d "${dev}" >> -   done >> -   rm -f "$(_nvme_def_file_path)" >> -} >> - >> -_cleanup_nvmet() { >> -   local dev >> -   local port >> -   local subsys >> -   local transport >> -   local name >> - >> -   if [[ ! -d "${NVMET_CFS}" ]]; then >> -      return 0 >> -   fi >> - >> -   # Don't let successive Ctrl-Cs interrupt the cleanup processes >> -   trap '' SIGINT >> - >> -   shopt -s nullglob >> - >> -   for dev in /sys/class/nvme/nvme*; do >> -      dev="$(basename "$dev")" >> -      transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" >> -      if [[ "$transport" == "${nvme_trtype}" ]]; then >> -         # if udev auto connect is enabled for FC we get false positives >> -         if [[ "$transport" != "fc" ]]; then >> -            echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" >> -         fi >> -         _nvme_disconnect_ctrl "${dev}" 2>/dev/null >> -      fi >> -   done >> - >> -   for port in "${NVMET_CFS}"/ports/*; do >> -      name=$(basename "${port}") >> -      echo "WARNING: Test did not clean up port: ${name}" >> -      rm -f "${port}"/subsystems/* >> -      rmdir "${port}" >> -   done >> - >> -   for subsys in "${NVMET_CFS}"/subsystems/*; do >> -      name=$(basename "${subsys}") >> -      echo "WARNING: Test did not clean up subsystem: ${name}" >> -      for ns in "${subsys}"/namespaces/*; do >> -         rmdir "${ns}" >> -      done >> -      rmdir "${subsys}" >> -   done >> - >> -   for host in "${NVMET_CFS}"/hosts/*; do >> -      name=$(basename "${host}") >> -      echo "WARNING: Test did not clean up host: ${name}" >> -      rmdir "${host}" >> -   done >> - >> -   shopt -u nullglob >> -   trap SIGINT >> - >> -   if [[ "${nvme_trtype}" == "fc" ]]; then >> -      _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ >> -            "${def_remote_wwnn}" "${def_remote_wwpn}" >> -      modprobe -rq nvme-fcloop 2>/dev/null >> -   fi >> -   modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null >> -   if [[ "${nvme_trtype}" != "loop" ]]; then >> -      modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null >> -   fi >> -   modprobe -rq nvmet 2>/dev/null >> -   if [[ "${nvme_trtype}" == "rdma" ]]; then >> -      stop_soft_rdma >> -   fi >> - >> -   _cleanup_blkdev >> -} >> - >> -_setup_nvmet() { >> -   _register_test_cleanup _cleanup_nvmet >> -   modprobe -q nvmet >> -   if [[ "${nvme_trtype}" != "loop" ]]; then >> -      modprobe -q nvmet-"${nvme_trtype}" >> -   fi >> -   modprobe -q nvme-"${nvme_trtype}" >> -   if [[ "${nvme_trtype}" == "rdma" ]]; then >> -      start_soft_rdma >> -      for i in $(rdma_network_interfaces) >> -      do >> -         if [[ "${nvme_adrfam}" == "ipv6" ]]; then >> -            ipv6_addr=$(get_ipv6_ll_addr "$i") >> -            if [[ -n "${ipv6_addr}" ]]; then >> -               def_traddr=${ipv6_addr} >> -            fi >> -         else >> -            ipv4_addr=$(get_ipv4_addr "$i") >> -            if [[ -n "${ipv4_addr}" ]]; then >> -               def_traddr=${ipv4_addr} >> -            fi >> -         fi >> -      done >> -   fi >> -   if [[ "${nvme_trtype}" = "fc" ]]; then >> -      modprobe -q nvme-fcloop >> -      _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ >> -              "${def_remote_wwnn}" "${def_remote_wwpn}" >> - >> -      def_traddr=$(printf "nn-%s:pn-%s" \ >> -               "${def_remote_wwnn}" \ >> -               "${def_remote_wwpn}") >> -      def_host_traddr=$(printf "nn-%s:pn-%s" \ >> -               "${def_local_wwnn}" \ >> -               "${def_local_wwpn}") >> -   fi >> -} >> - >> -_nvme_disconnect_ctrl() { >> -   local ctrl="$1" >> - >> -   nvme disconnect --device "${ctrl}" >> -} >> - >> _nvme_disconnect_subsys() { >>     local subsysnqn="$def_subsysnqn" >>  >> @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() { >>        grep -o "disconnected.*" >> } >>  >> -_nvme_connect_subsys() { >> -   local subsysnqn="$def_subsysnqn" >> -   local hostnqn="$def_hostnqn" > > It looks weird that _nvme_connect_subsys() is moved to common/nvme, but > _nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to > move _nvme_disconnect_subsys() also. Currently, md/001 does not use > _nvme_disconnect_subsys(). Isn't it the better to call it in > cleanup_nvme_over_tcp()? > I agree, I would move _nvme_disconnect_subsys() in v3. cleanup_nvme_over_tcp() use _nvme_disconnect_ctrl() to disconnect the controller by the controller name rather the subsys name. I can change it to _nvme_disconnect_subsys() if it's more appropriate. >> -   local hostid="$def_hostid" >> -   local hostkey="" >> -   local ctrlkey="" >> -   local nr_io_queues="" >> -   local nr_write_queues="" >> -   local nr_poll_queues="" >> -   local keep_alive_tmo="" >> -   local reconnect_delay="" >> -   local ctrl_loss_tmo="" >> -   local no_wait=false >> -   local i >> - >> -   while [[ $# -gt 0 ]]; do >> -      case $1 in >> -         --subsysnqn) >> -            subsysnqn="$2" >> -            shift 2 >> -            ;; >> -         --hostnqn) >> -            hostnqn="$2" >> -            shift 2 >> -            ;; >> -         --hostid) >> -            hostid="$2" >> -            shift 2 >> -            ;; >> -         --dhchap-secret) >> -            hostkey="$2" >> -            shift 2 >> -            ;; >> -         --dhchap-ctrl-secret) >> -            ctrlkey="$2" >> -            shift 2 >> -            ;; >> -         --nr-io-queues) >> -            nr_io_queues="$2" >> -            shift 2 >> -            ;; >> -         --nr-write-queues) >> -            nr_write_queues="$2" >> -            shift 2 >> -            ;; >> -         --nr-poll-queues) >> -            nr_poll_queues="$2" >> -            shift 2 >> -            ;; >> -         --keep-alive-tmo) >> -            keep_alive_tmo="$2" >> -            shift 2 >> -            ;; >> -         --reconnect-delay) >> -            reconnect_delay="$2" >> -            shift 2 >> -            ;; >> -         --ctrl-loss-tmo) >> -            ctrl_loss_tmo="$2" >> -            shift 2 >> -            ;; >> -         --no-wait) >> -            no_wait=true >> -            shift 1 >> -            ;; >> -         *) >> -            echo "WARNING: unknown argument: $1" >> -            shift >> -            ;; >> -      esac >> -   done >> - >> -   ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") >> -   if [[ "${nvme_trtype}" == "fc" ]] ; then >> -      ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") >> -   elif [[ "${nvme_trtype}" != "loop" ]]; then >> -      ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") >> -   fi >> -   ARGS+=(--hostnqn="${hostnqn}") >> -   ARGS+=(--hostid="${hostid}") >> -   if [[ -n "${hostkey}" ]]; then >> -      ARGS+=(--dhchap-secret="${hostkey}") >> -   fi >> -   if [[ -n "${ctrlkey}" ]]; then >> -      ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") >> -   fi >> -   if [[ -n "${nr_io_queues}" ]]; then >> -      ARGS+=(--nr-io-queues="${nr_io_queues}") >> -   fi >> -   if [[ -n "${nr_write_queues}" ]]; then >> -      ARGS+=(--nr-write-queues="${nr_write_queues}") >> -   fi >> -   if [[ -n "${nr_poll_queues}" ]]; then >> -      ARGS+=(--nr-poll-queues="${nr_poll_queues}") >> -   fi >> -   if [[ -n "${keep_alive_tmo}" ]]; then >> -      ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") >> -   fi >> -   if [[ -n "${reconnect_delay}" ]]; then >> -      ARGS+=(--reconnect-delay="${reconnect_delay}") >> -   fi >> -   if [[ -n "${ctrl_loss_tmo}" ]]; then >> -      ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") >> -   fi >> - >> -   nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" >> - >> -   # Wait until device file and uuid/wwid sysfs attributes get ready for >> -   # all namespaces. >> -   if [[ ${no_wait} = false ]]; then >> -      udevadm settle >> -      for ((i = 0; i < 10; i++)); do >> -         _nvme_ns_ready "${subsysnqn}" && return >> -         sleep .1 >> -      done >> -   fi >> -} >> - >> -_nvme_ns_ready() { >> -   local subsysnqn="${1}" >> -   local ns_path ns_id dev >> -   local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" >> - >> -   dev=$(_find_nvme_dev "$subsysnqn") >> -   for ns_path in "${cfs_path}/namespaces/"*; do >> -      ns_id=${ns_path##*/} >> -      if [[ ! -b /dev/${dev}n${ns_id} || >> -           ! -e /sys/block/${dev}n${ns_id}/uuid || >> -           ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then >> -         return 1 >> -      fi >> -   done >> -   return 0 >> -} >> - >> _nvme_discover() { >>     local trtype="$1" >>     local traddr="${2:-$def_traddr}" >> @@ -617,73 +223,6 @@ _nvme_discover() { >>     nvme discover "${ARGS[@]}" >> } >>  >> -_create_nvmet_port() { >> -   local trtype="$1" >> -   local traddr="${2:-$def_traddr}" >> -   local adrfam="${3:-$def_adrfam}" >> -   local trsvcid="${4:-$def_trsvcid}" >> - >> -   local port >> -   for ((port = 0; ; port++)); do >> -      if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then >> -         break >> -      fi >> -   done >> - >> -   mkdir "${NVMET_CFS}/ports/${port}" >> -   echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" >> -   echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" >> -   echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" >> -   if [[ "${adrfam}" != "fc" ]]; then >> -      echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" >> -   fi >> - >> -   echo "${port}" >> -} >> - >> -_remove_nvmet_port() { >> -   local port="$1" >> -   rmdir "${NVMET_CFS}/ports/${port}" >> -} >> - >> -_create_nvmet_ns() { >> -   local nvmet_subsystem="$1" >> -   local nsid="$2" >> -   local blkdev="$3" >> -   local uuid="00000000-0000-0000-0000-000000000000" >> -   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> -   local ns_path="${subsys_path}/namespaces/${nsid}" >> - >> -   if [[ $# -eq 4 ]]; then >> -      uuid="$4" >> -   fi >> - >> -   mkdir "${ns_path}" >> -   printf "%s" "${blkdev}" > "${ns_path}/device_path" >> -   printf "%s" "${uuid}" > "${ns_path}/device_uuid" >> -   printf 1 > "${ns_path}/enable" >> -} >> - >> -_create_nvmet_subsystem() { >> -   local nvmet_subsystem="$1" >> -   local blkdev="$2" >> -   local uuid=$3 >> -   local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> - >> -   mkdir -p "${cfs_path}" >> -   echo 0 > "${cfs_path}/attr_allow_any_host" >> -   _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" >> -} >> - >> -_add_nvmet_allow_hosts() { >> -   local nvmet_subsystem="$1" >> -   local nvmet_hostnqn="$2" >> -   local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> -   local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" >> - >> -   ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" >> -} >> - >> _remove_nvmet_allow_hosts() { >>     local nvmet_subsystem="$1" >>     local nvmet_hostnqn="$2" >> @@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() { >>     rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" >> } >>  >> -_create_nvmet_host() { >> -   local nvmet_subsystem="$1" >> -   local nvmet_hostnqn="$2" >> -   local nvmet_hostkey="$3" >> -   local nvmet_ctrlkey="$4" >> -   local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" >> - >> -   if [[ -d "${host_path}" ]]; then >> -      echo "FAIL target setup failed. stale host configuration found" >> -      return 1; >> -   fi >> - >> -   mkdir "${host_path}" >> -   _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" >> -   if [[ "${nvmet_hostkey}" ]] ; then >> -      echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" >> -   fi >> -   if [[ "${nvmet_ctrlkey}" ]] ; then >> -      echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" >> -   fi >> -} >> - >> -_remove_nvmet_ns() { >> -   local nvmet_subsystem="$1" >> -   local nsid=$2 >> -   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> -   local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" >> - >> -   echo 0 > "${nvmet_ns_path}/enable" >> -   rmdir "${nvmet_ns_path}" >> -} >> - >> -_remove_nvmet_subsystem() { >> -   local nvmet_subsystem="$1" >> -   local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> - >> -   _remove_nvmet_ns "${nvmet_subsystem}" "1" >> -   rm -f "${subsys_path}"/allowed_hosts/* >> -   rmdir "${subsys_path}" >> -} >> - >> -_remove_nvmet_host() { >> -   local nvmet_host="$1" >> -   local host_path="${NVMET_CFS}/hosts/${nvmet_host}" >> - >> -   rmdir "${host_path}" >> -} >> - >> _create_nvmet_passthru() { >>     local nvmet_subsystem="$1" >>     local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" >> @@ -765,34 +256,6 @@ _remove_nvmet_passhtru() { >>     rmdir "${subsys_path}" >> } >>  >> -_add_nvmet_subsys_to_port() { >> -   local port="$1" >> -   local nvmet_subsystem="$2" >> - >> -   ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ >> -      "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" >> -} >> - >> -_remove_nvmet_subsystem_from_port() { >> -   local port="$1" >> -   local nvmet_subsystem="$2" >> - >> -   rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" >> -} >> - >> -_get_nvmet_ports() { >> -   local nvmet_subsystem="$1" >> -   local -n nvmet_ports="$2" >> -   local cfs_path="${NVMET_CFS}/ports" >> -   local sarg >> - >> -   sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" >> - >> -   for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do >> -      nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") >> -   done >> -} >> - >> _set_nvmet_hostkey() { >>     local nvmet_hostnqn="$1" >>     local nvmet_hostkey="$2" >> @@ -829,20 +292,6 @@ _set_nvmet_dhgroup() { >>         "${cfs_path}/dhchap_dhgroup" >> } >>  >> -_find_nvme_dev() { >> -   local subsys=$1 >> -   local subsysnqn >> -   local dev >> -   for dev in /sys/class/nvme/nvme*; do >> -      [ -e "$dev" ] || continue >> -      dev="$(basename "$dev")" >> -      subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" >> -      if [[ "$subsysnqn" == "$subsys" ]]; then >> -         echo "$dev" >> -      fi >> -   done >> -} >> - >> _find_nvme_ns() { >>     local subsys_uuid=$1 >>     local uuid >> @@ -924,44 +373,6 @@ _nvmet_target_setup() { >>           "${hostkey}" "${ctrlkey}" >> } >>  >> -_nvmet_target_cleanup() { >> -   local ports >> -   local port >> -   local blkdev >> -   local subsysnqn="${def_subsysnqn}" >> -   local blkdev_type="" >> - >> -   while [[ $# -gt 0 ]]; do >> -      case $1 in >> -         --blkdev) >> -            blkdev_type="$2" >> -            shift 2 >> -            ;; >> -         --subsysnqn) >> -            subsysnqn="$2" >> -            shift 2 >> -            ;; >> -         *) >> -            echo "WARNING: unknown argument: $1" >> -            shift >> -            ;; >> -      esac >> -   done >> - >> -   _get_nvmet_ports "${subsysnqn}" ports >> - >> -   for port in "${ports[@]}"; do >> -      _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" >> -      _remove_nvmet_port "${port}" >> -   done >> -   _remove_nvmet_subsystem "${subsysnqn}" >> -   _remove_nvmet_host "${def_hostnqn}" >> - >> -   if [[ "${blkdev_type}" == "device" ]]; then >> -      _cleanup_blkdev >> -   fi >> -} >> - >> _nvmet_passthru_target_setup() { >>     local subsysnqn="$def_subsysnqn" >>     local port >> -- >> 2.45.1
On 7/16/24 12:23, Ofir Gal wrote: >>> -_nvme_disconnect_ctrl() { >>> -Â Â Â local ctrl="$1" >>> - >>> -Â Â Â nvme disconnect --device "${ctrl}" >>> -} >>> - >>>Â Â _nvme_disconnect_subsys() { >>>Â Â Â Â Â local subsysnqn="$def_subsysnqn" >>> Â >>> @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() { >>>Â Â Â Â Â Â Â Â Â grep -o "disconnected.*" >>>Â Â } >>> Â >>> -_nvme_connect_subsys() { >>> -Â Â Â local subsysnqn="$def_subsysnqn" >>> -Â Â Â local hostnqn="$def_hostnqn" >> >> It looks weird that _nvme_connect_subsys() is moved to common/nvme, but >> _nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to >> move _nvme_disconnect_subsys() also. Currently, md/001 does not use >> _nvme_disconnect_subsys(). Isn't it the better to call it in >> cleanup_nvme_over_tcp()? >> > I agree, I would move _nvme_disconnect_subsys() in v3. > cleanup_nvme_over_tcp() use _nvme_disconnect_ctrl() to disconnect the > controller by the controller name rather the subsys name. I can change > it to _nvme_disconnect_subsys() if it's more appropriate. > I changed it to _nvme_disconnect_subsys() after using _find_nvme_ns() as Daniel suggested, will be in v3.
diff --git a/common/nvme b/common/nvme new file mode 100644 index 0000000..1800263 --- /dev/null +++ b/common/nvme @@ -0,0 +1,595 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# +# nvme helper functions. + +. common/shellcheck + +def_traddr="127.0.0.1" +def_adrfam="ipv4" +def_trsvcid="4420" +def_remote_wwnn="0x10001100aa000001" +def_remote_wwpn="0x20001100aa000001" +def_local_wwnn="0x10001100aa000002" +def_local_wwpn="0x20001100aa000002" +def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" +def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" +export def_subsysnqn="blktests-subsystem-1" +export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" +_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" +_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G +_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 +nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} +NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} +NVMET_CFS="/sys/kernel/config/nvmet/" + +# TMPDIR can not be referred out of test() or test_device() context. Instead of +# global variable def_flie_path, use this getter function. +_nvme_def_file_path() { + echo "${TMPDIR}/img" +} + +_require_nvme_trtype() { + local trtype + for trtype in "$@"; do + if [[ "${nvme_trtype}" == "$trtype" ]]; then + return 0 + fi + done + SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") + return 1 +} + +_require_nvme_trtype_is_loop() { + if ! _require_nvme_trtype loop; then + return 1 + fi + return 0 +} + +_require_nvme_trtype_is_fabrics() { + if ! _require_nvme_trtype loop fc rdma tcp; then + return 1 + fi + return 0 +} + +_nvme_fcloop_add_rport() { + local local_wwnn="$1" + local local_wwpn="$2" + local remote_wwnn="$3" + local remote_wwpn="$4" + local loopctl=/sys/class/fcloop/ctl + + echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port +} + +_nvme_fcloop_add_lport() { + local wwnn="$1" + local wwpn="$2" + local loopctl=/sys/class/fcloop/ctl + + echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port +} + +_nvme_fcloop_add_tport() { + local wwnn="$1" + local wwpn="$2" + local loopctl=/sys/class/fcloop/ctl + + echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port +} + +_setup_fcloop() { + local local_wwnn="${1:-$def_local_wwnn}" + local local_wwpn="${2:-$def_local_wwpn}" + local remote_wwnn="${3:-$def_remote_wwnn}" + local remote_wwpn="${4:-$def_remote_wwpn}" + + _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" + _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" + _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ + "${remote_wwnn}" "${remote_wwpn}" +} + +_nvme_fcloop_del_rport() { + local local_wwnn="$1" + local local_wwpn="$2" + local remote_wwnn="$3" + local remote_wwpn="$4" + local loopctl=/sys/class/fcloop/ctl + + if [[ ! -f "${loopctl}/del_remote_port" ]]; then + return + fi + echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" +} + +_nvme_fcloop_del_lport() { + local wwnn="$1" + local wwpn="$2" + local loopctl=/sys/class/fcloop/ctl + + if [[ ! -f "${loopctl}/del_local_port" ]]; then + return + fi + echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" +} + +_nvme_fcloop_del_tport() { + local wwnn="$1" + local wwpn="$2" + local loopctl=/sys/class/fcloop/ctl + + if [[ ! -f "${loopctl}/del_target_port" ]]; then + return + fi + echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" +} + +_cleanup_fcloop() { + local local_wwnn="${1:-$def_local_wwnn}" + local local_wwpn="${2:-$def_local_wwpn}" + local remote_wwnn="${3:-$def_remote_wwnn}" + local remote_wwpn="${4:-$def_remote_wwpn}" + + _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" + _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" + _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ + "${remote_wwnn}" "${remote_wwpn}" +} + +_cleanup_blkdev() { + local blkdev + local dev + + blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" + for dev in ${blkdev}; do + losetup -d "${dev}" + done + rm -f "$(_nvme_def_file_path)" +} + +_cleanup_nvmet() { + local dev + local port + local subsys + local transport + local name + + if [[ ! -d "${NVMET_CFS}" ]]; then + return 0 + fi + + # Don't let successive Ctrl-Cs interrupt the cleanup processes + trap '' SIGINT + + shopt -s nullglob + + for dev in /sys/class/nvme/nvme*; do + dev="$(basename "$dev")" + transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" + if [[ "$transport" == "${nvme_trtype}" ]]; then + # if udev auto connect is enabled for FC we get false positives + if [[ "$transport" != "fc" ]]; then + echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" + fi + _nvme_disconnect_ctrl "${dev}" 2>/dev/null + fi + done + + for port in "${NVMET_CFS}"/ports/*; do + name=$(basename "${port}") + echo "WARNING: Test did not clean up port: ${name}" + rm -f "${port}"/subsystems/* + rmdir "${port}" + done + + for subsys in "${NVMET_CFS}"/subsystems/*; do + name=$(basename "${subsys}") + echo "WARNING: Test did not clean up subsystem: ${name}" + for ns in "${subsys}"/namespaces/*; do + rmdir "${ns}" + done + rmdir "${subsys}" + done + + for host in "${NVMET_CFS}"/hosts/*; do + name=$(basename "${host}") + echo "WARNING: Test did not clean up host: ${name}" + rmdir "${host}" + done + + shopt -u nullglob + trap SIGINT + + if [[ "${nvme_trtype}" == "fc" ]]; then + _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ + "${def_remote_wwnn}" "${def_remote_wwpn}" + modprobe -rq nvme-fcloop 2>/dev/null + fi + modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null + if [[ "${nvme_trtype}" != "loop" ]]; then + modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null + fi + modprobe -rq nvmet 2>/dev/null + if [[ "${nvme_trtype}" == "rdma" ]]; then + stop_soft_rdma + fi + + _cleanup_blkdev +} + +_setup_nvmet() { + _register_test_cleanup _cleanup_nvmet + modprobe -q nvmet + if [[ "${nvme_trtype}" != "loop" ]]; then + modprobe -q nvmet-"${nvme_trtype}" + fi + modprobe -q nvme-"${nvme_trtype}" + if [[ "${nvme_trtype}" == "rdma" ]]; then + start_soft_rdma + for i in $(rdma_network_interfaces) + do + if [[ "${nvme_adrfam}" == "ipv6" ]]; then + ipv6_addr=$(get_ipv6_ll_addr "$i") + if [[ -n "${ipv6_addr}" ]]; then + def_traddr=${ipv6_addr} + fi + else + ipv4_addr=$(get_ipv4_addr "$i") + if [[ -n "${ipv4_addr}" ]]; then + def_traddr=${ipv4_addr} + fi + fi + done + fi + if [[ "${nvme_trtype}" = "fc" ]]; then + modprobe -q nvme-fcloop + _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ + "${def_remote_wwnn}" "${def_remote_wwpn}" + + def_traddr=$(printf "nn-%s:pn-%s" \ + "${def_remote_wwnn}" \ + "${def_remote_wwpn}") + def_host_traddr=$(printf "nn-%s:pn-%s" \ + "${def_local_wwnn}" \ + "${def_local_wwpn}") + fi +} + +_nvme_disconnect_ctrl() { + local ctrl="$1" + + nvme disconnect --device "${ctrl}" +} + +_nvme_connect_subsys() { + local subsysnqn="$def_subsysnqn" + local hostnqn="$def_hostnqn" + local hostid="$def_hostid" + local hostkey="" + local ctrlkey="" + local nr_io_queues="" + local nr_write_queues="" + local nr_poll_queues="" + local keep_alive_tmo="" + local reconnect_delay="" + local ctrl_loss_tmo="" + local no_wait=false + local i + + while [[ $# -gt 0 ]]; do + case $1 in + --subsysnqn) + subsysnqn="$2" + shift 2 + ;; + --hostnqn) + hostnqn="$2" + shift 2 + ;; + --hostid) + hostid="$2" + shift 2 + ;; + --dhchap-secret) + hostkey="$2" + shift 2 + ;; + --dhchap-ctrl-secret) + ctrlkey="$2" + shift 2 + ;; + --nr-io-queues) + nr_io_queues="$2" + shift 2 + ;; + --nr-write-queues) + nr_write_queues="$2" + shift 2 + ;; + --nr-poll-queues) + nr_poll_queues="$2" + shift 2 + ;; + --keep-alive-tmo) + keep_alive_tmo="$2" + shift 2 + ;; + --reconnect-delay) + reconnect_delay="$2" + shift 2 + ;; + --ctrl-loss-tmo) + ctrl_loss_tmo="$2" + shift 2 + ;; + --no-wait) + no_wait=true + shift 1 + ;; + *) + echo "WARNING: unknown argument: $1" + shift + ;; + esac + done + + ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") + if [[ "${nvme_trtype}" == "fc" ]] ; then + ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") + elif [[ "${nvme_trtype}" != "loop" ]]; then + ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") + fi + ARGS+=(--hostnqn="${hostnqn}") + ARGS+=(--hostid="${hostid}") + if [[ -n "${hostkey}" ]]; then + ARGS+=(--dhchap-secret="${hostkey}") + fi + if [[ -n "${ctrlkey}" ]]; then + ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") + fi + if [[ -n "${nr_io_queues}" ]]; then + ARGS+=(--nr-io-queues="${nr_io_queues}") + fi + if [[ -n "${nr_write_queues}" ]]; then + ARGS+=(--nr-write-queues="${nr_write_queues}") + fi + if [[ -n "${nr_poll_queues}" ]]; then + ARGS+=(--nr-poll-queues="${nr_poll_queues}") + fi + if [[ -n "${keep_alive_tmo}" ]]; then + ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") + fi + if [[ -n "${reconnect_delay}" ]]; then + ARGS+=(--reconnect-delay="${reconnect_delay}") + fi + if [[ -n "${ctrl_loss_tmo}" ]]; then + ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") + fi + + nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" + + # Wait until device file and uuid/wwid sysfs attributes get ready for + # all namespaces. + if [[ ${no_wait} = false ]]; then + udevadm settle + for ((i = 0; i < 10; i++)); do + _nvme_ns_ready "${subsysnqn}" && return + sleep .1 + done + fi +} + +_nvme_ns_ready() { + local subsysnqn="${1}" + local ns_path ns_id dev + local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" + + dev=$(_find_nvme_dev "$subsysnqn") + for ns_path in "${cfs_path}/namespaces/"*; do + ns_id=${ns_path##*/} + if [[ ! -b /dev/${dev}n${ns_id} || + ! -e /sys/block/${dev}n${ns_id}/uuid || + ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then + return 1 + fi + done + return 0 +} + +_create_nvmet_port() { + local trtype="$1" + local traddr="${2:-$def_traddr}" + local adrfam="${3:-$def_adrfam}" + local trsvcid="${4:-$def_trsvcid}" + + local port + for ((port = 0; ; port++)); do + if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then + break + fi + done + + mkdir "${NVMET_CFS}/ports/${port}" + echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" + echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" + echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" + if [[ "${adrfam}" != "fc" ]]; then + echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" + fi + + echo "${port}" +} + +_remove_nvmet_port() { + local port="$1" + rmdir "${NVMET_CFS}/ports/${port}" +} + +_create_nvmet_ns() { + local nvmet_subsystem="$1" + local nsid="$2" + local blkdev="$3" + local uuid="00000000-0000-0000-0000-000000000000" + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + local ns_path="${subsys_path}/namespaces/${nsid}" + + if [[ $# -eq 4 ]]; then + uuid="$4" + fi + + mkdir "${ns_path}" + printf "%s" "${blkdev}" > "${ns_path}/device_path" + printf "%s" "${uuid}" > "${ns_path}/device_uuid" + printf 1 > "${ns_path}/enable" +} + +_create_nvmet_subsystem() { + local nvmet_subsystem="$1" + local blkdev="$2" + local uuid=$3 + local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + + mkdir -p "${cfs_path}" + echo 0 > "${cfs_path}/attr_allow_any_host" + _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" +} + +_add_nvmet_allow_hosts() { + local nvmet_subsystem="$1" + local nvmet_hostnqn="$2" + local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" + + ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" +} + +_create_nvmet_host() { + local nvmet_subsystem="$1" + local nvmet_hostnqn="$2" + local nvmet_hostkey="$3" + local nvmet_ctrlkey="$4" + local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" + + if [[ -d "${host_path}" ]]; then + echo "FAIL target setup failed. stale host configuration found" + return 1; + fi + + mkdir "${host_path}" + _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" + if [[ "${nvmet_hostkey}" ]] ; then + echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" + fi + if [[ "${nvmet_ctrlkey}" ]] ; then + echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" + fi +} + +_remove_nvmet_ns() { + local nvmet_subsystem="$1" + local nsid=$2 + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" + + echo 0 > "${nvmet_ns_path}/enable" + rmdir "${nvmet_ns_path}" +} + +_remove_nvmet_subsystem() { + local nvmet_subsystem="$1" + local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + + _remove_nvmet_ns "${nvmet_subsystem}" "1" + rm -f "${subsys_path}"/allowed_hosts/* + rmdir "${subsys_path}" +} + +_remove_nvmet_host() { + local nvmet_host="$1" + local host_path="${NVMET_CFS}/hosts/${nvmet_host}" + + rmdir "${host_path}" +} + +_add_nvmet_subsys_to_port() { + local port="$1" + local nvmet_subsystem="$2" + + ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ + "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" +} + +_remove_nvmet_subsystem_from_port() { + local port="$1" + local nvmet_subsystem="$2" + + rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" +} + +_get_nvmet_ports() { + local nvmet_subsystem="$1" + local -n nvmet_ports="$2" + local cfs_path="${NVMET_CFS}/ports" + local sarg + + sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" + + for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do + nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") + done +} + +_find_nvme_dev() { + local subsys=$1 + local subsysnqn + local dev + for dev in /sys/class/nvme/nvme*; do + [ -e "$dev" ] || continue + dev="$(basename "$dev")" + subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" + if [[ "$subsysnqn" == "$subsys" ]]; then + echo "$dev" + fi + done +} + +_nvmet_target_cleanup() { + local ports + local port + local blkdev + local subsysnqn="${def_subsysnqn}" + local blkdev_type="" + + while [[ $# -gt 0 ]]; do + case $1 in + --blkdev) + blkdev_type="$2" + shift 2 + ;; + --subsysnqn) + subsysnqn="$2" + shift 2 + ;; + *) + echo "WARNING: unknown argument: $1" + shift + ;; + esac + done + + _get_nvmet_ports "${subsysnqn}" ports + + for port in "${ports[@]}"; do + _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" + _remove_nvmet_port "${port}" + done + _remove_nvmet_subsystem "${subsysnqn}" + _remove_nvmet_host "${def_hostnqn}" + + if [[ "${blkdev_type}" == "device" ]]; then + _cleanup_blkdev + fi +} diff --git a/tests/nvme/rc b/tests/nvme/rc index c1ddf41..3462f2e 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -5,25 +5,9 @@ # Test specific to NVMe devices . common/rc +. common/nvme . common/multipath-over-rdma -def_traddr="127.0.0.1" -def_adrfam="ipv4" -def_trsvcid="4420" -def_remote_wwnn="0x10001100aa000001" -def_remote_wwpn="0x20001100aa000001" -def_local_wwnn="0x10001100aa000002" -def_local_wwpn="0x20001100aa000002" -def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" -def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" -export def_subsysnqn="blktests-subsystem-1" -export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e" -_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop" -_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G -_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000 -nvmet_blkdev_type=${nvmet_blkdev_type:-"device"} -NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"} - _NVMET_TRTYPES_is_valid() { local type @@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() { COND_DESC="bd=${nvmet_blkdev_type}" } -# TMPDIR can not be referred out of test() or test_device() context. Instead of -# global variable def_flie_path, use this getter function. -_nvme_def_file_path() { - echo "${TMPDIR}/img" -} - _nvme_requires() { _have_program nvme _require_nvme_test_img_size 4m @@ -144,8 +122,6 @@ group_device_requires() { _require_test_dev_is_nvme } -NVMET_CFS="/sys/kernel/config/nvmet/" - _require_test_dev_is_nvme() { if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then SKIP_REASONS+=("$TEST_DEV is not a NVMe device") @@ -168,31 +144,6 @@ _require_nvme_test_img_size() { return 0 } -_require_nvme_trtype() { - local trtype - for trtype in "$@"; do - if [[ "${nvme_trtype}" == "$trtype" ]]; then - return 0 - fi - done - SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test") - return 1 -} - -_require_nvme_trtype_is_loop() { - if ! _require_nvme_trtype loop; then - return 1 - fi - return 0 -} - -_require_nvme_trtype_is_fabrics() { - if ! _require_nvme_trtype loop fc rdma tcp; then - return 1 - fi - return 0 -} - _require_nvme_cli_auth() { if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then SKIP_REASONS+=("nvme gen-dhchap-key command missing") @@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() { echo "${io_size_kb}k" } -_nvme_fcloop_add_rport() { - local local_wwnn="$1" - local local_wwpn="$2" - local remote_wwnn="$3" - local remote_wwpn="$4" - local loopctl=/sys/class/fcloop/ctl - - echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port -} - -_nvme_fcloop_add_lport() { - local wwnn="$1" - local wwpn="$2" - local loopctl=/sys/class/fcloop/ctl - - echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port -} - -_nvme_fcloop_add_tport() { - local wwnn="$1" - local wwpn="$2" - local loopctl=/sys/class/fcloop/ctl - - echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port -} - -_setup_fcloop() { - local local_wwnn="${1:-$def_local_wwnn}" - local local_wwpn="${2:-$def_local_wwpn}" - local remote_wwnn="${3:-$def_remote_wwnn}" - local remote_wwpn="${4:-$def_remote_wwpn}" - - _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" - _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" - _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ - "${remote_wwnn}" "${remote_wwpn}" -} - -_nvme_fcloop_del_rport() { - local local_wwnn="$1" - local local_wwpn="$2" - local remote_wwnn="$3" - local remote_wwpn="$4" - local loopctl=/sys/class/fcloop/ctl - - if [[ ! -f "${loopctl}/del_remote_port" ]]; then - return - fi - echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port" -} - -_nvme_fcloop_del_lport() { - local wwnn="$1" - local wwpn="$2" - local loopctl=/sys/class/fcloop/ctl - - if [[ ! -f "${loopctl}/del_local_port" ]]; then - return - fi - echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port" -} - -_nvme_fcloop_del_tport() { - local wwnn="$1" - local wwpn="$2" - local loopctl=/sys/class/fcloop/ctl - - if [[ ! -f "${loopctl}/del_target_port" ]]; then - return - fi - echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" -} - -_cleanup_fcloop() { - local local_wwnn="${1:-$def_local_wwnn}" - local local_wwpn="${2:-$def_local_wwpn}" - local remote_wwnn="${3:-$def_remote_wwnn}" - local remote_wwpn="${4:-$def_remote_wwpn}" - - _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" - _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" - _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ - "${remote_wwnn}" "${remote_wwpn}" -} - -_cleanup_blkdev() { - local blkdev - local dev - - blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')" - for dev in ${blkdev}; do - losetup -d "${dev}" - done - rm -f "$(_nvme_def_file_path)" -} - -_cleanup_nvmet() { - local dev - local port - local subsys - local transport - local name - - if [[ ! -d "${NVMET_CFS}" ]]; then - return 0 - fi - - # Don't let successive Ctrl-Cs interrupt the cleanup processes - trap '' SIGINT - - shopt -s nullglob - - for dev in /sys/class/nvme/nvme*; do - dev="$(basename "$dev")" - transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)" - if [[ "$transport" == "${nvme_trtype}" ]]; then - # if udev auto connect is enabled for FC we get false positives - if [[ "$transport" != "fc" ]]; then - echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}" - fi - _nvme_disconnect_ctrl "${dev}" 2>/dev/null - fi - done - - for port in "${NVMET_CFS}"/ports/*; do - name=$(basename "${port}") - echo "WARNING: Test did not clean up port: ${name}" - rm -f "${port}"/subsystems/* - rmdir "${port}" - done - - for subsys in "${NVMET_CFS}"/subsystems/*; do - name=$(basename "${subsys}") - echo "WARNING: Test did not clean up subsystem: ${name}" - for ns in "${subsys}"/namespaces/*; do - rmdir "${ns}" - done - rmdir "${subsys}" - done - - for host in "${NVMET_CFS}"/hosts/*; do - name=$(basename "${host}") - echo "WARNING: Test did not clean up host: ${name}" - rmdir "${host}" - done - - shopt -u nullglob - trap SIGINT - - if [[ "${nvme_trtype}" == "fc" ]]; then - _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ - "${def_remote_wwnn}" "${def_remote_wwpn}" - modprobe -rq nvme-fcloop 2>/dev/null - fi - modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null - if [[ "${nvme_trtype}" != "loop" ]]; then - modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null - fi - modprobe -rq nvmet 2>/dev/null - if [[ "${nvme_trtype}" == "rdma" ]]; then - stop_soft_rdma - fi - - _cleanup_blkdev -} - -_setup_nvmet() { - _register_test_cleanup _cleanup_nvmet - modprobe -q nvmet - if [[ "${nvme_trtype}" != "loop" ]]; then - modprobe -q nvmet-"${nvme_trtype}" - fi - modprobe -q nvme-"${nvme_trtype}" - if [[ "${nvme_trtype}" == "rdma" ]]; then - start_soft_rdma - for i in $(rdma_network_interfaces) - do - if [[ "${nvme_adrfam}" == "ipv6" ]]; then - ipv6_addr=$(get_ipv6_ll_addr "$i") - if [[ -n "${ipv6_addr}" ]]; then - def_traddr=${ipv6_addr} - fi - else - ipv4_addr=$(get_ipv4_addr "$i") - if [[ -n "${ipv4_addr}" ]]; then - def_traddr=${ipv4_addr} - fi - fi - done - fi - if [[ "${nvme_trtype}" = "fc" ]]; then - modprobe -q nvme-fcloop - _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ - "${def_remote_wwnn}" "${def_remote_wwpn}" - - def_traddr=$(printf "nn-%s:pn-%s" \ - "${def_remote_wwnn}" \ - "${def_remote_wwpn}") - def_host_traddr=$(printf "nn-%s:pn-%s" \ - "${def_local_wwnn}" \ - "${def_local_wwpn}") - fi -} - -_nvme_disconnect_ctrl() { - local ctrl="$1" - - nvme disconnect --device "${ctrl}" -} - _nvme_disconnect_subsys() { local subsysnqn="$def_subsysnqn" @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() { grep -o "disconnected.*" } -_nvme_connect_subsys() { - local subsysnqn="$def_subsysnqn" - local hostnqn="$def_hostnqn" - local hostid="$def_hostid" - local hostkey="" - local ctrlkey="" - local nr_io_queues="" - local nr_write_queues="" - local nr_poll_queues="" - local keep_alive_tmo="" - local reconnect_delay="" - local ctrl_loss_tmo="" - local no_wait=false - local i - - while [[ $# -gt 0 ]]; do - case $1 in - --subsysnqn) - subsysnqn="$2" - shift 2 - ;; - --hostnqn) - hostnqn="$2" - shift 2 - ;; - --hostid) - hostid="$2" - shift 2 - ;; - --dhchap-secret) - hostkey="$2" - shift 2 - ;; - --dhchap-ctrl-secret) - ctrlkey="$2" - shift 2 - ;; - --nr-io-queues) - nr_io_queues="$2" - shift 2 - ;; - --nr-write-queues) - nr_write_queues="$2" - shift 2 - ;; - --nr-poll-queues) - nr_poll_queues="$2" - shift 2 - ;; - --keep-alive-tmo) - keep_alive_tmo="$2" - shift 2 - ;; - --reconnect-delay) - reconnect_delay="$2" - shift 2 - ;; - --ctrl-loss-tmo) - ctrl_loss_tmo="$2" - shift 2 - ;; - --no-wait) - no_wait=true - shift 1 - ;; - *) - echo "WARNING: unknown argument: $1" - shift - ;; - esac - done - - ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") - if [[ "${nvme_trtype}" == "fc" ]] ; then - ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") - elif [[ "${nvme_trtype}" != "loop" ]]; then - ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") - fi - ARGS+=(--hostnqn="${hostnqn}") - ARGS+=(--hostid="${hostid}") - if [[ -n "${hostkey}" ]]; then - ARGS+=(--dhchap-secret="${hostkey}") - fi - if [[ -n "${ctrlkey}" ]]; then - ARGS+=(--dhchap-ctrl-secret="${ctrlkey}") - fi - if [[ -n "${nr_io_queues}" ]]; then - ARGS+=(--nr-io-queues="${nr_io_queues}") - fi - if [[ -n "${nr_write_queues}" ]]; then - ARGS+=(--nr-write-queues="${nr_write_queues}") - fi - if [[ -n "${nr_poll_queues}" ]]; then - ARGS+=(--nr-poll-queues="${nr_poll_queues}") - fi - if [[ -n "${keep_alive_tmo}" ]]; then - ARGS+=(--keep-alive-tmo="${keep_alive_tmo}") - fi - if [[ -n "${reconnect_delay}" ]]; then - ARGS+=(--reconnect-delay="${reconnect_delay}") - fi - if [[ -n "${ctrl_loss_tmo}" ]]; then - ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") - fi - - nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" - - # Wait until device file and uuid/wwid sysfs attributes get ready for - # all namespaces. - if [[ ${no_wait} = false ]]; then - udevadm settle - for ((i = 0; i < 10; i++)); do - _nvme_ns_ready "${subsysnqn}" && return - sleep .1 - done - fi -} - -_nvme_ns_ready() { - local subsysnqn="${1}" - local ns_path ns_id dev - local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" - - dev=$(_find_nvme_dev "$subsysnqn") - for ns_path in "${cfs_path}/namespaces/"*; do - ns_id=${ns_path##*/} - if [[ ! -b /dev/${dev}n${ns_id} || - ! -e /sys/block/${dev}n${ns_id}/uuid || - ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then - return 1 - fi - done - return 0 -} - _nvme_discover() { local trtype="$1" local traddr="${2:-$def_traddr}" @@ -617,73 +223,6 @@ _nvme_discover() { nvme discover "${ARGS[@]}" } -_create_nvmet_port() { - local trtype="$1" - local traddr="${2:-$def_traddr}" - local adrfam="${3:-$def_adrfam}" - local trsvcid="${4:-$def_trsvcid}" - - local port - for ((port = 0; ; port++)); do - if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then - break - fi - done - - mkdir "${NVMET_CFS}/ports/${port}" - echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" - echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" - echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" - if [[ "${adrfam}" != "fc" ]]; then - echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" - fi - - echo "${port}" -} - -_remove_nvmet_port() { - local port="$1" - rmdir "${NVMET_CFS}/ports/${port}" -} - -_create_nvmet_ns() { - local nvmet_subsystem="$1" - local nsid="$2" - local blkdev="$3" - local uuid="00000000-0000-0000-0000-000000000000" - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - local ns_path="${subsys_path}/namespaces/${nsid}" - - if [[ $# -eq 4 ]]; then - uuid="$4" - fi - - mkdir "${ns_path}" - printf "%s" "${blkdev}" > "${ns_path}/device_path" - printf "%s" "${uuid}" > "${ns_path}/device_uuid" - printf 1 > "${ns_path}/enable" -} - -_create_nvmet_subsystem() { - local nvmet_subsystem="$1" - local blkdev="$2" - local uuid=$3 - local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - - mkdir -p "${cfs_path}" - echo 0 > "${cfs_path}/attr_allow_any_host" - _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" -} - -_add_nvmet_allow_hosts() { - local nvmet_subsystem="$1" - local nvmet_hostnqn="$2" - local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" - - ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" -} - _remove_nvmet_allow_hosts() { local nvmet_subsystem="$1" local nvmet_hostnqn="$2" @@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() { rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}" } -_create_nvmet_host() { - local nvmet_subsystem="$1" - local nvmet_hostnqn="$2" - local nvmet_hostkey="$3" - local nvmet_ctrlkey="$4" - local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}" - - if [[ -d "${host_path}" ]]; then - echo "FAIL target setup failed. stale host configuration found" - return 1; - fi - - mkdir "${host_path}" - _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}" - if [[ "${nvmet_hostkey}" ]] ; then - echo "${nvmet_hostkey}" > "${host_path}/dhchap_key" - fi - if [[ "${nvmet_ctrlkey}" ]] ; then - echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key" - fi -} - -_remove_nvmet_ns() { - local nvmet_subsystem="$1" - local nsid=$2 - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - local nvmet_ns_path="${subsys_path}/namespaces/${nsid}" - - echo 0 > "${nvmet_ns_path}/enable" - rmdir "${nvmet_ns_path}" -} - -_remove_nvmet_subsystem() { - local nvmet_subsystem="$1" - local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - - _remove_nvmet_ns "${nvmet_subsystem}" "1" - rm -f "${subsys_path}"/allowed_hosts/* - rmdir "${subsys_path}" -} - -_remove_nvmet_host() { - local nvmet_host="$1" - local host_path="${NVMET_CFS}/hosts/${nvmet_host}" - - rmdir "${host_path}" -} - _create_nvmet_passthru() { local nvmet_subsystem="$1" local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" @@ -765,34 +256,6 @@ _remove_nvmet_passhtru() { rmdir "${subsys_path}" } -_add_nvmet_subsys_to_port() { - local port="$1" - local nvmet_subsystem="$2" - - ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \ - "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" -} - -_remove_nvmet_subsystem_from_port() { - local port="$1" - local nvmet_subsystem="$2" - - rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}" -} - -_get_nvmet_ports() { - local nvmet_subsystem="$1" - local -n nvmet_ports="$2" - local cfs_path="${NVMET_CFS}/ports" - local sarg - - sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" - - for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do - nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") - done -} - _set_nvmet_hostkey() { local nvmet_hostnqn="$1" local nvmet_hostkey="$2" @@ -829,20 +292,6 @@ _set_nvmet_dhgroup() { "${cfs_path}/dhchap_dhgroup" } -_find_nvme_dev() { - local subsys=$1 - local subsysnqn - local dev - for dev in /sys/class/nvme/nvme*; do - [ -e "$dev" ] || continue - dev="$(basename "$dev")" - subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" - if [[ "$subsysnqn" == "$subsys" ]]; then - echo "$dev" - fi - done -} - _find_nvme_ns() { local subsys_uuid=$1 local uuid @@ -924,44 +373,6 @@ _nvmet_target_setup() { "${hostkey}" "${ctrlkey}" } -_nvmet_target_cleanup() { - local ports - local port - local blkdev - local subsysnqn="${def_subsysnqn}" - local blkdev_type="" - - while [[ $# -gt 0 ]]; do - case $1 in - --blkdev) - blkdev_type="$2" - shift 2 - ;; - --subsysnqn) - subsysnqn="$2" - shift 2 - ;; - *) - echo "WARNING: unknown argument: $1" - shift - ;; - esac - done - - _get_nvmet_ports "${subsysnqn}" ports - - for port in "${ports[@]}"; do - _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" - _remove_nvmet_port "${port}" - done - _remove_nvmet_subsystem "${subsysnqn}" - _remove_nvmet_host "${def_hostnqn}" - - if [[ "${blkdev_type}" == "device" ]]; then - _cleanup_blkdev - fi -} - _nvmet_passthru_target_setup() { local subsysnqn="$def_subsysnqn" local port
Move functions from tests/nvme/rc to common/nvme to be able to reuse them in other tests groups. Signed-off-by: Ofir Gal <ofir.gal@volumez.com> --- I have moved the function that are nessecary for the regression test I add. Let me know if we want to move more or all the functions to common/nvme. shellcheck detects 2 new warnings: common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154] common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154] I have tried to figure out why but I couldn't figure out what is different than the upstream version. How can I fix them? common/nvme | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/nvme/rc | 591 +------------------------------------------------ 2 files changed, 596 insertions(+), 590 deletions(-) create mode 100644 common/nvme