new file mode 100644
@@ -0,0 +1,636 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+#
+# nvme helper functions.
+
+. common/shellcheck
+
+def_traddr="127.0.0.1"
+def_adrfam="ipv4"
+def_trsvcid="4420"
+def_remote_wwnn="0x10001100aa000001"
+def_remote_wwpn="0x20001100aa000001"
+def_local_wwnn="0x10001100aa000002"
+def_local_wwpn="0x20001100aa000002"
+def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
+def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
+export def_subsysnqn="blktests-subsystem-1"
+export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
+_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
+_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
+_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
+nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
+NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
+NVMET_CFS="/sys/kernel/config/nvmet/"
+nvme_trtype=${nvme_trtype:-}
+nvme_adrfam=${nvme_adrfam:-}
+
+# TMPDIR can not be referred out of test() or test_device() context. Instead of
+# global variable def_flie_path, use this getter function.
+_nvme_def_file_path() {
+ echo "${TMPDIR}/img"
+}
+
+_require_nvme_trtype() {
+ local trtype
+ for trtype in "$@"; do
+ if [[ "${nvme_trtype}" == "$trtype" ]]; then
+ return 0
+ fi
+ done
+ SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
+ return 1
+}
+
+_require_nvme_trtype_is_loop() {
+ if ! _require_nvme_trtype loop; then
+ return 1
+ fi
+ return 0
+}
+
+_require_nvme_trtype_is_fabrics() {
+ if ! _require_nvme_trtype loop fc rdma tcp; then
+ return 1
+ fi
+ return 0
+}
+
+_nvme_fcloop_add_rport() {
+ local local_wwnn="$1"
+ local local_wwpn="$2"
+ local remote_wwnn="$3"
+ local remote_wwpn="$4"
+ local loopctl=/sys/class/fcloop/ctl
+
+ echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
+}
+
+_nvme_fcloop_add_lport() {
+ local wwnn="$1"
+ local wwpn="$2"
+ local loopctl=/sys/class/fcloop/ctl
+
+ echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
+}
+
+_nvme_fcloop_add_tport() {
+ local wwnn="$1"
+ local wwpn="$2"
+ local loopctl=/sys/class/fcloop/ctl
+
+ echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
+}
+
+_setup_fcloop() {
+ local local_wwnn="${1:-$def_local_wwnn}"
+ local local_wwpn="${2:-$def_local_wwpn}"
+ local remote_wwnn="${3:-$def_remote_wwnn}"
+ local remote_wwpn="${4:-$def_remote_wwpn}"
+
+ _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
+ _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
+ _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
+ "${remote_wwnn}" "${remote_wwpn}"
+}
+
+_nvme_fcloop_del_rport() {
+ local local_wwnn="$1"
+ local local_wwpn="$2"
+ local remote_wwnn="$3"
+ local remote_wwpn="$4"
+ local loopctl=/sys/class/fcloop/ctl
+
+ if [[ ! -f "${loopctl}/del_remote_port" ]]; then
+ return
+ fi
+ echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
+}
+
+_nvme_fcloop_del_lport() {
+ local wwnn="$1"
+ local wwpn="$2"
+ local loopctl=/sys/class/fcloop/ctl
+
+ if [[ ! -f "${loopctl}/del_local_port" ]]; then
+ return
+ fi
+ echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
+}
+
+_nvme_fcloop_del_tport() {
+ local wwnn="$1"
+ local wwpn="$2"
+ local loopctl=/sys/class/fcloop/ctl
+
+ if [[ ! -f "${loopctl}/del_target_port" ]]; then
+ return
+ fi
+ echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
+}
+
+_cleanup_fcloop() {
+ local local_wwnn="${1:-$def_local_wwnn}"
+ local local_wwpn="${2:-$def_local_wwpn}"
+ local remote_wwnn="${3:-$def_remote_wwnn}"
+ local remote_wwpn="${4:-$def_remote_wwpn}"
+
+ _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
+ _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
+ _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
+ "${remote_wwnn}" "${remote_wwpn}"
+}
+
+_cleanup_blkdev() {
+ local blkdev
+ local dev
+
+ blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
+ for dev in ${blkdev}; do
+ losetup -d "${dev}"
+ done
+ rm -f "$(_nvme_def_file_path)"
+}
+
+_cleanup_nvmet() {
+ local dev
+ local port
+ local subsys
+ local transport
+ local name
+
+ if [[ ! -d "${NVMET_CFS}" ]]; then
+ return 0
+ fi
+
+ # Don't let successive Ctrl-Cs interrupt the cleanup processes
+ trap '' SIGINT
+
+ shopt -s nullglob
+
+ for dev in /sys/class/nvme/nvme*; do
+ dev="$(basename "$dev")"
+ transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
+ if [[ "$transport" == "${nvme_trtype}" ]]; then
+ # if udev auto connect is enabled for FC we get false positives
+ if [[ "$transport" != "fc" ]]; then
+ echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
+ fi
+ _nvme_disconnect_ctrl "${dev}" 2>/dev/null
+ fi
+ done
+
+ for port in "${NVMET_CFS}"/ports/*; do
+ name=$(basename "${port}")
+ echo "WARNING: Test did not clean up port: ${name}"
+ rm -f "${port}"/subsystems/*
+ rmdir "${port}"
+ done
+
+ for subsys in "${NVMET_CFS}"/subsystems/*; do
+ name=$(basename "${subsys}")
+ echo "WARNING: Test did not clean up subsystem: ${name}"
+ for ns in "${subsys}"/namespaces/*; do
+ rmdir "${ns}"
+ done
+ rmdir "${subsys}"
+ done
+
+ for host in "${NVMET_CFS}"/hosts/*; do
+ name=$(basename "${host}")
+ echo "WARNING: Test did not clean up host: ${name}"
+ rmdir "${host}"
+ done
+
+ shopt -u nullglob
+ trap SIGINT
+
+ if [[ "${nvme_trtype}" == "fc" ]]; then
+ _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
+ "${def_remote_wwnn}" "${def_remote_wwpn}"
+ modprobe -rq nvme-fcloop 2>/dev/null
+ fi
+ modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
+ if [[ "${nvme_trtype}" != "loop" ]]; then
+ modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
+ fi
+ modprobe -rq nvmet 2>/dev/null
+ if [[ "${nvme_trtype}" == "rdma" ]]; then
+ stop_soft_rdma
+ fi
+
+ _cleanup_blkdev
+}
+
+_setup_nvmet() {
+ _register_test_cleanup _cleanup_nvmet
+ modprobe -q nvmet
+ if [[ "${nvme_trtype}" != "loop" ]]; then
+ modprobe -q nvmet-"${nvme_trtype}"
+ fi
+ modprobe -q nvme-"${nvme_trtype}"
+ if [[ "${nvme_trtype}" == "rdma" ]]; then
+ start_soft_rdma
+ for i in $(rdma_network_interfaces)
+ do
+ if [[ "${nvme_adrfam}" == "ipv6" ]]; then
+ ipv6_addr=$(get_ipv6_ll_addr "$i")
+ if [[ -n "${ipv6_addr}" ]]; then
+ def_traddr=${ipv6_addr}
+ fi
+ else
+ ipv4_addr=$(get_ipv4_addr "$i")
+ if [[ -n "${ipv4_addr}" ]]; then
+ def_traddr=${ipv4_addr}
+ fi
+ fi
+ done
+ fi
+ if [[ "${nvme_trtype}" = "fc" ]]; then
+ modprobe -q nvme-fcloop
+ _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
+ "${def_remote_wwnn}" "${def_remote_wwpn}"
+
+ def_traddr=$(printf "nn-%s:pn-%s" \
+ "${def_remote_wwnn}" \
+ "${def_remote_wwpn}")
+ def_host_traddr=$(printf "nn-%s:pn-%s" \
+ "${def_local_wwnn}" \
+ "${def_local_wwpn}")
+ fi
+}
+
+_nvme_disconnect_ctrl() {
+ local ctrl="$1"
+
+ nvme disconnect --device "${ctrl}"
+}
+
+_nvme_connect_subsys() {
+ local subsysnqn="$def_subsysnqn"
+ local hostnqn="$def_hostnqn"
+ local hostid="$def_hostid"
+ local hostkey=""
+ local ctrlkey=""
+ local nr_io_queues=""
+ local nr_write_queues=""
+ local nr_poll_queues=""
+ local keep_alive_tmo=""
+ local reconnect_delay=""
+ local ctrl_loss_tmo=""
+ local no_wait=false
+ local i
+
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ --subsysnqn)
+ subsysnqn="$2"
+ shift 2
+ ;;
+ --hostnqn)
+ hostnqn="$2"
+ shift 2
+ ;;
+ --hostid)
+ hostid="$2"
+ shift 2
+ ;;
+ --dhchap-secret)
+ hostkey="$2"
+ shift 2
+ ;;
+ --dhchap-ctrl-secret)
+ ctrlkey="$2"
+ shift 2
+ ;;
+ --nr-io-queues)
+ nr_io_queues="$2"
+ shift 2
+ ;;
+ --nr-write-queues)
+ nr_write_queues="$2"
+ shift 2
+ ;;
+ --nr-poll-queues)
+ nr_poll_queues="$2"
+ shift 2
+ ;;
+ --keep-alive-tmo)
+ keep_alive_tmo="$2"
+ shift 2
+ ;;
+ --reconnect-delay)
+ reconnect_delay="$2"
+ shift 2
+ ;;
+ --ctrl-loss-tmo)
+ ctrl_loss_tmo="$2"
+ shift 2
+ ;;
+ --no-wait)
+ no_wait=true
+ shift 1
+ ;;
+ *)
+ echo "WARNING: unknown argument: $1"
+ shift
+ ;;
+ esac
+ done
+
+ ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
+ if [[ "${nvme_trtype}" == "fc" ]] ; then
+ ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
+ elif [[ "${nvme_trtype}" != "loop" ]]; then
+ ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
+ fi
+ ARGS+=(--hostnqn="${hostnqn}")
+ ARGS+=(--hostid="${hostid}")
+ if [[ -n "${hostkey}" ]]; then
+ ARGS+=(--dhchap-secret="${hostkey}")
+ fi
+ if [[ -n "${ctrlkey}" ]]; then
+ ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
+ fi
+ if [[ -n "${nr_io_queues}" ]]; then
+ ARGS+=(--nr-io-queues="${nr_io_queues}")
+ fi
+ if [[ -n "${nr_write_queues}" ]]; then
+ ARGS+=(--nr-write-queues="${nr_write_queues}")
+ fi
+ if [[ -n "${nr_poll_queues}" ]]; then
+ ARGS+=(--nr-poll-queues="${nr_poll_queues}")
+ fi
+ if [[ -n "${keep_alive_tmo}" ]]; then
+ ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
+ fi
+ if [[ -n "${reconnect_delay}" ]]; then
+ ARGS+=(--reconnect-delay="${reconnect_delay}")
+ fi
+ if [[ -n "${ctrl_loss_tmo}" ]]; then
+ ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
+ fi
+
+ nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
+
+ # Wait until device file and uuid/wwid sysfs attributes get ready for
+ # all namespaces.
+ if [[ ${no_wait} = false ]]; then
+ udevadm settle
+ for ((i = 0; i < 10; i++)); do
+ _nvme_ns_ready "${subsysnqn}" && return
+ sleep .1
+ done
+ fi
+}
+
+_nvme_disconnect_subsys() {
+ local subsysnqn="$def_subsysnqn"
+
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ --subsysnqn)
+ subsysnqn="$2"
+ shift 2
+ ;;
+ *)
+ echo "WARNING: unknown argument: $1"
+ shift
+ ;;
+ esac
+ done
+
+ nvme disconnect --nqn "${subsysnqn}" |& tee -a "$FULL" |
+ grep -o "disconnected.*"
+}
+
+
+_nvme_ns_ready() {
+ local subsysnqn="${1}"
+ local ns_path ns_id dev
+ local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
+
+ dev=$(_find_nvme_dev "$subsysnqn")
+ for ns_path in "${cfs_path}/namespaces/"*; do
+ ns_id=${ns_path##*/}
+ if [[ ! -b /dev/${dev}n${ns_id} ||
+ ! -e /sys/block/${dev}n${ns_id}/uuid ||
+ ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
+ return 1
+ fi
+ done
+ return 0
+}
+
+_create_nvmet_port() {
+ local trtype="$1"
+ local traddr="${2:-$def_traddr}"
+ local adrfam="${3:-$def_adrfam}"
+ local trsvcid="${4:-$def_trsvcid}"
+
+ local port
+ for ((port = 0; ; port++)); do
+ if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
+ break
+ fi
+ done
+
+ mkdir "${NVMET_CFS}/ports/${port}"
+ echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+ echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
+ echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
+ if [[ "${adrfam}" != "fc" ]]; then
+ echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
+ fi
+
+ echo "${port}"
+}
+
+_remove_nvmet_port() {
+ local port="$1"
+ rmdir "${NVMET_CFS}/ports/${port}"
+}
+
+_create_nvmet_ns() {
+ local nvmet_subsystem="$1"
+ local nsid="$2"
+ local blkdev="$3"
+ local uuid="00000000-0000-0000-0000-000000000000"
+ local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+ local ns_path="${subsys_path}/namespaces/${nsid}"
+
+ if [[ $# -eq 4 ]]; then
+ uuid="$4"
+ fi
+
+ mkdir "${ns_path}"
+ printf "%s" "${blkdev}" > "${ns_path}/device_path"
+ printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+ printf 1 > "${ns_path}/enable"
+}
+
+_create_nvmet_subsystem() {
+ local nvmet_subsystem="$1"
+ local blkdev="$2"
+ local uuid=$3
+ local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+
+ mkdir -p "${cfs_path}"
+ echo 0 > "${cfs_path}/attr_allow_any_host"
+ _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+}
+
+_add_nvmet_allow_hosts() {
+ local nvmet_subsystem="$1"
+ local nvmet_hostnqn="$2"
+ local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+ local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
+
+ ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
+}
+
+_create_nvmet_host() {
+ local nvmet_subsystem="$1"
+ local nvmet_hostnqn="$2"
+ local nvmet_hostkey="$3"
+ local nvmet_ctrlkey="$4"
+ local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
+
+ if [[ -d "${host_path}" ]]; then
+ echo "FAIL target setup failed. stale host configuration found"
+ return 1;
+ fi
+
+ mkdir "${host_path}"
+ _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
+ if [[ "${nvmet_hostkey}" ]] ; then
+ echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
+ fi
+ if [[ "${nvmet_ctrlkey}" ]] ; then
+ echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
+ fi
+}
+
+_remove_nvmet_ns() {
+ local nvmet_subsystem="$1"
+ local nsid=$2
+ local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+ local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
+
+ echo 0 > "${nvmet_ns_path}/enable"
+ rmdir "${nvmet_ns_path}"
+}
+
+_remove_nvmet_subsystem() {
+ local nvmet_subsystem="$1"
+ local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+
+ _remove_nvmet_ns "${nvmet_subsystem}" "1"
+ rm -f "${subsys_path}"/allowed_hosts/*
+ rmdir "${subsys_path}"
+}
+
+_remove_nvmet_host() {
+ local nvmet_host="$1"
+ local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
+
+ rmdir "${host_path}"
+}
+
+_add_nvmet_subsys_to_port() {
+ local port="$1"
+ local nvmet_subsystem="$2"
+
+ ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
+ "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
+}
+
+_remove_nvmet_subsystem_from_port() {
+ local port="$1"
+ local nvmet_subsystem="$2"
+
+ rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
+}
+
+_get_nvmet_ports() {
+ local nvmet_subsystem="$1"
+ local -n nvmet_ports="$2"
+ local cfs_path="${NVMET_CFS}/ports"
+ local sarg
+
+ sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
+
+ for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
+ nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
+ done
+}
+
+_find_nvme_dev() {
+ local subsys=$1
+ local subsysnqn
+ local dev
+ for dev in /sys/class/nvme/nvme*; do
+ [ -e "$dev" ] || continue
+ dev="$(basename "$dev")"
+ subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
+ if [[ "$subsysnqn" == "$subsys" ]]; then
+ echo "$dev"
+ fi
+ done
+}
+
+_find_nvme_ns() {
+ local subsys_uuid=$1
+ local uuid
+ local ns
+
+ for ns in "/sys/block/nvme"* ; do
+ # ignore nvme channel block devices
+ if ! [[ "${ns}" =~ nvme[0-9]+n[0-9]+ ]]; then
+ continue
+ fi
+ [ -e "${ns}/uuid" ] || continue
+ uuid=$(cat "${ns}/uuid")
+ if [[ "${subsys_uuid}" == "${uuid}" ]]; then
+ basename "${ns}"
+ fi
+ done
+}
+
+_nvmet_target_cleanup() {
+ local ports
+ local port
+ local blkdev
+ local subsysnqn="${def_subsysnqn}"
+ local blkdev_type=""
+
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ --blkdev)
+ blkdev_type="$2"
+ shift 2
+ ;;
+ --subsysnqn)
+ subsysnqn="$2"
+ shift 2
+ ;;
+ *)
+ echo "WARNING: unknown argument: $1"
+ shift
+ ;;
+ esac
+ done
+
+ _get_nvmet_ports "${subsysnqn}" ports
+
+ for port in "${ports[@]}"; do
+ _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
+ _remove_nvmet_port "${port}"
+ done
+ _remove_nvmet_subsystem "${subsysnqn}"
+ _remove_nvmet_host "${def_hostnqn}"
+
+ if [[ "${blkdev_type}" == "device" ]]; then
+ _cleanup_blkdev
+ fi
+}
@@ -5,25 +5,9 @@
# Test specific to NVMe devices
. common/rc
+. common/nvme
. common/multipath-over-rdma
-def_traddr="127.0.0.1"
-def_adrfam="ipv4"
-def_trsvcid="4420"
-def_remote_wwnn="0x10001100aa000001"
-def_remote_wwpn="0x20001100aa000001"
-def_local_wwnn="0x10001100aa000002"
-def_local_wwpn="0x20001100aa000002"
-def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
-def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
-export def_subsysnqn="blktests-subsystem-1"
-export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
-_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
-_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
-_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
-nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
-NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
-
_NVMET_TRTYPES_is_valid() {
local type
@@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() {
COND_DESC="bd=${nvmet_blkdev_type}"
}
-# TMPDIR can not be referred out of test() or test_device() context. Instead of
-# global variable def_flie_path, use this getter function.
-_nvme_def_file_path() {
- echo "${TMPDIR}/img"
-}
-
_nvme_requires() {
_have_program nvme
_require_nvme_test_img_size 4m
@@ -144,8 +122,6 @@ group_device_requires() {
_require_test_dev_is_nvme
}
-NVMET_CFS="/sys/kernel/config/nvmet/"
-
_require_test_dev_is_nvme() {
if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
SKIP_REASONS+=("$TEST_DEV is not a NVMe device")
@@ -168,31 +144,6 @@ _require_nvme_test_img_size() {
return 0
}
-_require_nvme_trtype() {
- local trtype
- for trtype in "$@"; do
- if [[ "${nvme_trtype}" == "$trtype" ]]; then
- return 0
- fi
- done
- SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
- return 1
-}
-
-_require_nvme_trtype_is_loop() {
- if ! _require_nvme_trtype loop; then
- return 1
- fi
- return 0
-}
-
-_require_nvme_trtype_is_fabrics() {
- if ! _require_nvme_trtype loop fc rdma tcp; then
- return 1
- fi
- return 0
-}
-
_require_nvme_cli_auth() {
if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then
SKIP_REASONS+=("nvme gen-dhchap-key command missing")
@@ -235,371 +186,6 @@ _nvme_calc_rand_io_size() {
echo "${io_size_kb}k"
}
-_nvme_fcloop_add_rport() {
- local local_wwnn="$1"
- local local_wwpn="$2"
- local remote_wwnn="$3"
- local remote_wwpn="$4"
- local loopctl=/sys/class/fcloop/ctl
-
- echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
-}
-
-_nvme_fcloop_add_lport() {
- local wwnn="$1"
- local wwpn="$2"
- local loopctl=/sys/class/fcloop/ctl
-
- echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
-}
-
-_nvme_fcloop_add_tport() {
- local wwnn="$1"
- local wwpn="$2"
- local loopctl=/sys/class/fcloop/ctl
-
- echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
-}
-
-_setup_fcloop() {
- local local_wwnn="${1:-$def_local_wwnn}"
- local local_wwpn="${2:-$def_local_wwpn}"
- local remote_wwnn="${3:-$def_remote_wwnn}"
- local remote_wwpn="${4:-$def_remote_wwpn}"
-
- _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
- _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
- _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
- "${remote_wwnn}" "${remote_wwpn}"
-}
-
-_nvme_fcloop_del_rport() {
- local local_wwnn="$1"
- local local_wwpn="$2"
- local remote_wwnn="$3"
- local remote_wwpn="$4"
- local loopctl=/sys/class/fcloop/ctl
-
- if [[ ! -f "${loopctl}/del_remote_port" ]]; then
- return
- fi
- echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
-}
-
-_nvme_fcloop_del_lport() {
- local wwnn="$1"
- local wwpn="$2"
- local loopctl=/sys/class/fcloop/ctl
-
- if [[ ! -f "${loopctl}/del_local_port" ]]; then
- return
- fi
- echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
-}
-
-_nvme_fcloop_del_tport() {
- local wwnn="$1"
- local wwpn="$2"
- local loopctl=/sys/class/fcloop/ctl
-
- if [[ ! -f "${loopctl}/del_target_port" ]]; then
- return
- fi
- echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
-}
-
-_cleanup_fcloop() {
- local local_wwnn="${1:-$def_local_wwnn}"
- local local_wwpn="${2:-$def_local_wwpn}"
- local remote_wwnn="${3:-$def_remote_wwnn}"
- local remote_wwpn="${4:-$def_remote_wwpn}"
-
- _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
- _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
- _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
- "${remote_wwnn}" "${remote_wwpn}"
-}
-
-_cleanup_blkdev() {
- local blkdev
- local dev
-
- blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
- for dev in ${blkdev}; do
- losetup -d "${dev}"
- done
- rm -f "$(_nvme_def_file_path)"
-}
-
-_cleanup_nvmet() {
- local dev
- local port
- local subsys
- local transport
- local name
-
- if [[ ! -d "${NVMET_CFS}" ]]; then
- return 0
- fi
-
- # Don't let successive Ctrl-Cs interrupt the cleanup processes
- trap '' SIGINT
-
- shopt -s nullglob
-
- for dev in /sys/class/nvme/nvme*; do
- dev="$(basename "$dev")"
- transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
- if [[ "$transport" == "${nvme_trtype}" ]]; then
- # if udev auto connect is enabled for FC we get false positives
- if [[ "$transport" != "fc" ]]; then
- echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
- fi
- _nvme_disconnect_ctrl "${dev}" 2>/dev/null
- fi
- done
-
- for port in "${NVMET_CFS}"/ports/*; do
- name=$(basename "${port}")
- echo "WARNING: Test did not clean up port: ${name}"
- rm -f "${port}"/subsystems/*
- rmdir "${port}"
- done
-
- for subsys in "${NVMET_CFS}"/subsystems/*; do
- name=$(basename "${subsys}")
- echo "WARNING: Test did not clean up subsystem: ${name}"
- for ns in "${subsys}"/namespaces/*; do
- rmdir "${ns}"
- done
- rmdir "${subsys}"
- done
-
- for host in "${NVMET_CFS}"/hosts/*; do
- name=$(basename "${host}")
- echo "WARNING: Test did not clean up host: ${name}"
- rmdir "${host}"
- done
-
- shopt -u nullglob
- trap SIGINT
-
- if [[ "${nvme_trtype}" == "fc" ]]; then
- _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
- "${def_remote_wwnn}" "${def_remote_wwpn}"
- modprobe -rq nvme-fcloop 2>/dev/null
- fi
- modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
- if [[ "${nvme_trtype}" != "loop" ]]; then
- modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
- fi
- modprobe -rq nvmet 2>/dev/null
- if [[ "${nvme_trtype}" == "rdma" ]]; then
- stop_soft_rdma
- fi
-
- _cleanup_blkdev
-}
-
-_setup_nvmet() {
- _register_test_cleanup _cleanup_nvmet
- modprobe -q nvmet
- if [[ "${nvme_trtype}" != "loop" ]]; then
- modprobe -q nvmet-"${nvme_trtype}"
- fi
- modprobe -q nvme-"${nvme_trtype}"
- if [[ "${nvme_trtype}" == "rdma" ]]; then
- start_soft_rdma
- for i in $(rdma_network_interfaces)
- do
- if [[ "${nvme_adrfam}" == "ipv6" ]]; then
- ipv6_addr=$(get_ipv6_ll_addr "$i")
- if [[ -n "${ipv6_addr}" ]]; then
- def_traddr=${ipv6_addr}
- fi
- else
- ipv4_addr=$(get_ipv4_addr "$i")
- if [[ -n "${ipv4_addr}" ]]; then
- def_traddr=${ipv4_addr}
- fi
- fi
- done
- fi
- if [[ "${nvme_trtype}" = "fc" ]]; then
- modprobe -q nvme-fcloop
- _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
- "${def_remote_wwnn}" "${def_remote_wwpn}"
-
- def_traddr=$(printf "nn-%s:pn-%s" \
- "${def_remote_wwnn}" \
- "${def_remote_wwpn}")
- def_host_traddr=$(printf "nn-%s:pn-%s" \
- "${def_local_wwnn}" \
- "${def_local_wwpn}")
- fi
-}
-
-_nvme_disconnect_ctrl() {
- local ctrl="$1"
-
- nvme disconnect --device "${ctrl}"
-}
-
-_nvme_disconnect_subsys() {
- local subsysnqn="$def_subsysnqn"
-
- while [[ $# -gt 0 ]]; do
- case $1 in
- --subsysnqn)
- subsysnqn="$2"
- shift 2
- ;;
- *)
- echo "WARNING: unknown argument: $1"
- shift
- ;;
- esac
- done
-
- nvme disconnect --nqn "${subsysnqn}" |& tee -a "$FULL" |
- grep -o "disconnected.*"
-}
-
-_nvme_connect_subsys() {
- local subsysnqn="$def_subsysnqn"
- local hostnqn="$def_hostnqn"
- local hostid="$def_hostid"
- local hostkey=""
- local ctrlkey=""
- local nr_io_queues=""
- local nr_write_queues=""
- local nr_poll_queues=""
- local keep_alive_tmo=""
- local reconnect_delay=""
- local ctrl_loss_tmo=""
- local no_wait=false
- local i
-
- while [[ $# -gt 0 ]]; do
- case $1 in
- --subsysnqn)
- subsysnqn="$2"
- shift 2
- ;;
- --hostnqn)
- hostnqn="$2"
- shift 2
- ;;
- --hostid)
- hostid="$2"
- shift 2
- ;;
- --dhchap-secret)
- hostkey="$2"
- shift 2
- ;;
- --dhchap-ctrl-secret)
- ctrlkey="$2"
- shift 2
- ;;
- --nr-io-queues)
- nr_io_queues="$2"
- shift 2
- ;;
- --nr-write-queues)
- nr_write_queues="$2"
- shift 2
- ;;
- --nr-poll-queues)
- nr_poll_queues="$2"
- shift 2
- ;;
- --keep-alive-tmo)
- keep_alive_tmo="$2"
- shift 2
- ;;
- --reconnect-delay)
- reconnect_delay="$2"
- shift 2
- ;;
- --ctrl-loss-tmo)
- ctrl_loss_tmo="$2"
- shift 2
- ;;
- --no-wait)
- no_wait=true
- shift 1
- ;;
- *)
- echo "WARNING: unknown argument: $1"
- shift
- ;;
- esac
- done
-
- ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
- if [[ "${nvme_trtype}" == "fc" ]] ; then
- ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
- elif [[ "${nvme_trtype}" != "loop" ]]; then
- ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
- fi
- ARGS+=(--hostnqn="${hostnqn}")
- ARGS+=(--hostid="${hostid}")
- if [[ -n "${hostkey}" ]]; then
- ARGS+=(--dhchap-secret="${hostkey}")
- fi
- if [[ -n "${ctrlkey}" ]]; then
- ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
- fi
- if [[ -n "${nr_io_queues}" ]]; then
- ARGS+=(--nr-io-queues="${nr_io_queues}")
- fi
- if [[ -n "${nr_write_queues}" ]]; then
- ARGS+=(--nr-write-queues="${nr_write_queues}")
- fi
- if [[ -n "${nr_poll_queues}" ]]; then
- ARGS+=(--nr-poll-queues="${nr_poll_queues}")
- fi
- if [[ -n "${keep_alive_tmo}" ]]; then
- ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
- fi
- if [[ -n "${reconnect_delay}" ]]; then
- ARGS+=(--reconnect-delay="${reconnect_delay}")
- fi
- if [[ -n "${ctrl_loss_tmo}" ]]; then
- ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
- fi
-
- nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
-
- # Wait until device file and uuid/wwid sysfs attributes get ready for
- # all namespaces.
- if [[ ${no_wait} = false ]]; then
- udevadm settle
- for ((i = 0; i < 10; i++)); do
- _nvme_ns_ready "${subsysnqn}" && return
- sleep .1
- done
- fi
-}
-
-_nvme_ns_ready() {
- local subsysnqn="${1}"
- local ns_path ns_id dev
- local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
-
- dev=$(_find_nvme_dev "$subsysnqn")
- for ns_path in "${cfs_path}/namespaces/"*; do
- ns_id=${ns_path##*/}
- if [[ ! -b /dev/${dev}n${ns_id} ||
- ! -e /sys/block/${dev}n${ns_id}/uuid ||
- ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
- return 1
- fi
- done
- return 0
-}
-
_nvme_discover() {
local trtype="$1"
local traddr="${2:-$def_traddr}"
@@ -617,73 +203,6 @@ _nvme_discover() {
nvme discover "${ARGS[@]}"
}
-_create_nvmet_port() {
- local trtype="$1"
- local traddr="${2:-$def_traddr}"
- local adrfam="${3:-$def_adrfam}"
- local trsvcid="${4:-$def_trsvcid}"
-
- local port
- for ((port = 0; ; port++)); do
- if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
- break
- fi
- done
-
- mkdir "${NVMET_CFS}/ports/${port}"
- echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
- echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
- echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
- if [[ "${adrfam}" != "fc" ]]; then
- echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
- fi
-
- echo "${port}"
-}
-
-_remove_nvmet_port() {
- local port="$1"
- rmdir "${NVMET_CFS}/ports/${port}"
-}
-
-_create_nvmet_ns() {
- local nvmet_subsystem="$1"
- local nsid="$2"
- local blkdev="$3"
- local uuid="00000000-0000-0000-0000-000000000000"
- local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
- local ns_path="${subsys_path}/namespaces/${nsid}"
-
- if [[ $# -eq 4 ]]; then
- uuid="$4"
- fi
-
- mkdir "${ns_path}"
- printf "%s" "${blkdev}" > "${ns_path}/device_path"
- printf "%s" "${uuid}" > "${ns_path}/device_uuid"
- printf 1 > "${ns_path}/enable"
-}
-
-_create_nvmet_subsystem() {
- local nvmet_subsystem="$1"
- local blkdev="$2"
- local uuid=$3
- local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-
- mkdir -p "${cfs_path}"
- echo 0 > "${cfs_path}/attr_allow_any_host"
- _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
-}
-
-_add_nvmet_allow_hosts() {
- local nvmet_subsystem="$1"
- local nvmet_hostnqn="$2"
- local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
- local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
-
- ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
-}
-
_remove_nvmet_allow_hosts() {
local nvmet_subsystem="$1"
local nvmet_hostnqn="$2"
@@ -692,54 +211,6 @@ _remove_nvmet_allow_hosts() {
rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
}
-_create_nvmet_host() {
- local nvmet_subsystem="$1"
- local nvmet_hostnqn="$2"
- local nvmet_hostkey="$3"
- local nvmet_ctrlkey="$4"
- local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
-
- if [[ -d "${host_path}" ]]; then
- echo "FAIL target setup failed. stale host configuration found"
- return 1;
- fi
-
- mkdir "${host_path}"
- _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
- if [[ "${nvmet_hostkey}" ]] ; then
- echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
- fi
- if [[ "${nvmet_ctrlkey}" ]] ; then
- echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
- fi
-}
-
-_remove_nvmet_ns() {
- local nvmet_subsystem="$1"
- local nsid=$2
- local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
- local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
-
- echo 0 > "${nvmet_ns_path}/enable"
- rmdir "${nvmet_ns_path}"
-}
-
-_remove_nvmet_subsystem() {
- local nvmet_subsystem="$1"
- local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-
- _remove_nvmet_ns "${nvmet_subsystem}" "1"
- rm -f "${subsys_path}"/allowed_hosts/*
- rmdir "${subsys_path}"
-}
-
-_remove_nvmet_host() {
- local nvmet_host="$1"
- local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
-
- rmdir "${host_path}"
-}
-
_create_nvmet_passthru() {
local nvmet_subsystem="$1"
local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
@@ -765,34 +236,6 @@ _remove_nvmet_passhtru() {
rmdir "${subsys_path}"
}
-_add_nvmet_subsys_to_port() {
- local port="$1"
- local nvmet_subsystem="$2"
-
- ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
- "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
-}
-
-_remove_nvmet_subsystem_from_port() {
- local port="$1"
- local nvmet_subsystem="$2"
-
- rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
-}
-
-_get_nvmet_ports() {
- local nvmet_subsystem="$1"
- local -n nvmet_ports="$2"
- local cfs_path="${NVMET_CFS}/ports"
- local sarg
-
- sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
-
- for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
- nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
- done
-}
-
_set_nvmet_hostkey() {
local nvmet_hostnqn="$1"
local nvmet_hostkey="$2"
@@ -829,38 +272,6 @@ _set_nvmet_dhgroup() {
"${cfs_path}/dhchap_dhgroup"
}
-_find_nvme_dev() {
- local subsys=$1
- local subsysnqn
- local dev
- for dev in /sys/class/nvme/nvme*; do
- [ -e "$dev" ] || continue
- dev="$(basename "$dev")"
- subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
- if [[ "$subsysnqn" == "$subsys" ]]; then
- echo "$dev"
- fi
- done
-}
-
-_find_nvme_ns() {
- local subsys_uuid=$1
- local uuid
- local ns
-
- for ns in "/sys/block/nvme"* ; do
- # ignore nvme channel block devices
- if ! [[ "${ns}" =~ nvme[0-9]+n[0-9]+ ]]; then
- continue
- fi
- [ -e "${ns}/uuid" ] || continue
- uuid=$(cat "${ns}/uuid")
- if [[ "${subsys_uuid}" == "${uuid}" ]]; then
- basename "${ns}"
- fi
- done
-}
-
_find_nvme_passthru_loop_dev() {
local subsys=$1
local nsid
@@ -924,44 +335,6 @@ _nvmet_target_setup() {
"${hostkey}" "${ctrlkey}"
}
-_nvmet_target_cleanup() {
- local ports
- local port
- local blkdev
- local subsysnqn="${def_subsysnqn}"
- local blkdev_type=""
-
- while [[ $# -gt 0 ]]; do
- case $1 in
- --blkdev)
- blkdev_type="$2"
- shift 2
- ;;
- --subsysnqn)
- subsysnqn="$2"
- shift 2
- ;;
- *)
- echo "WARNING: unknown argument: $1"
- shift
- ;;
- esac
- done
-
- _get_nvmet_ports "${subsysnqn}" ports
-
- for port in "${ports[@]}"; do
- _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
- _remove_nvmet_port "${port}"
- done
- _remove_nvmet_subsystem "${subsysnqn}"
- _remove_nvmet_host "${def_hostnqn}"
-
- if [[ "${blkdev_type}" == "device" ]]; then
- _cleanup_blkdev
- fi
-}
-
_nvmet_passthru_target_setup() {
local subsysnqn="$def_subsysnqn"
local port
Move functions from tests/nvme/rc to common/nvme to be able to reuse them in other tests groups. Signed-off-by: Ofir Gal <ofir.gal@volumez.com> --- common/nvme | 636 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/nvme/rc | 629 +------------------------------------------------ 2 files changed, 637 insertions(+), 628 deletions(-) create mode 100644 common/nvme