@@ -328,7 +328,7 @@ static void target_shutdown_sessions(struct se_node_acl *acl)
restart:
spin_lock_irqsave(&acl->nacl_sess_lock, flags);
list_for_each_entry(sess, &acl->acl_sess_list, sess_acl_list) {
- if (atomic_read(&sess->stopped))
+ if (sess->cmd_cnt && atomic_read(&sess->cmd_cnt->stopped))
continue;
list_del_init(&sess->sess_acl_list);
@@ -220,11 +220,49 @@ void transport_subsystem_check_init(void)
sub_api_initialized = 1;
}
-static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
+static void target_release_cmd_refcnt(struct percpu_ref *ref)
{
- struct se_session *sess = container_of(ref, typeof(*sess), cmd_count);
+ struct target_cmd_counter *cmd_cnt = container_of(ref,
+ typeof(*cmd_cnt),
+ refcnt);
+ wake_up(&cmd_cnt->refcnt_wq);
+}
+
+static struct target_cmd_counter *target_alloc_cmd_counter(void)
+{
+ struct target_cmd_counter *cmd_cnt;
+ int rc;
+
+ cmd_cnt = kzalloc(sizeof(*cmd_cnt), GFP_KERNEL);
+ if (!cmd_cnt)
+ return NULL;
+
+ init_completion(&cmd_cnt->stop_done);
+ init_waitqueue_head(&cmd_cnt->refcnt_wq);
+ atomic_set(&cmd_cnt->stopped, 0);
+
+ rc = percpu_ref_init(&cmd_cnt->refcnt, target_release_cmd_refcnt, 0,
+ GFP_KERNEL);
+ if (rc)
+ goto free_cmd_cnt;
+
+ return cmd_cnt;
+
+free_cmd_cnt:
+ kfree(cmd_cnt);
+ return NULL;
+}
+
+static void target_free_cmd_counter(struct target_cmd_counter *cmd_cnt)
+{
+ /*
+ * Drivers like loop do not call target_stop_session during session
+ * shutdown so we have to drop the ref taken at init time here.
+ */
+ if (!atomic_read(&cmd_cnt->stopped))
+ percpu_ref_put(&cmd_cnt->refcnt);
- wake_up(&sess->cmd_count_wq);
+ percpu_ref_exit(&cmd_cnt->refcnt);
}
/**
@@ -238,25 +276,17 @@ int transport_init_session(struct se_session *se_sess)
INIT_LIST_HEAD(&se_sess->sess_list);
INIT_LIST_HEAD(&se_sess->sess_acl_list);
spin_lock_init(&se_sess->sess_cmd_lock);
- init_waitqueue_head(&se_sess->cmd_count_wq);
- init_completion(&se_sess->stop_done);
- atomic_set(&se_sess->stopped, 0);
- return percpu_ref_init(&se_sess->cmd_count,
- target_release_sess_cmd_refcnt, 0, GFP_KERNEL);
+ se_sess->cmd_cnt = target_alloc_cmd_counter();
+ if (!se_sess->cmd_cnt)
+ return -ENOMEM;
+
+ return 0;
}
EXPORT_SYMBOL(transport_init_session);
void transport_uninit_session(struct se_session *se_sess)
{
- /*
- * Drivers like iscsi and loop do not call target_stop_session
- * during session shutdown so we have to drop the ref taken at init
- * time here.
- */
- if (!atomic_read(&se_sess->stopped))
- percpu_ref_put(&se_sess->cmd_count);
-
- percpu_ref_exit(&se_sess->cmd_count);
+ target_free_cmd_counter(se_sess->cmd_cnt);
}
/**
@@ -602,7 +632,6 @@ void transport_free_session(struct se_session *se_sess)
sbitmap_queue_free(&se_sess->sess_tag_pool);
kvfree(se_sess->sess_cmd_map);
}
- transport_uninit_session(se_sess);
kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);
@@ -655,8 +684,13 @@ EXPORT_SYMBOL(transport_deregister_session);
void target_remove_session(struct se_session *se_sess)
{
+ struct target_cmd_counter *cmd_cnt = se_sess->cmd_cnt;
+
transport_deregister_session_configfs(se_sess);
transport_deregister_session(se_sess);
+
+ if (cmd_cnt)
+ target_free_cmd_counter(cmd_cnt);
}
EXPORT_SYMBOL(target_remove_session);
@@ -2970,9 +3004,16 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
se_cmd->se_cmd_flags |= SCF_ACK_KREF;
}
- if (!percpu_ref_tryget_live(&se_sess->cmd_count))
- ret = -ESHUTDOWN;
-
+ /*
+ * Users like xcopy do not use counters since they never do a stop
+ * and wait.
+ */
+ if (se_sess->cmd_cnt) {
+ if (!percpu_ref_tryget_live(&se_sess->cmd_cnt->refcnt))
+ ret = -ESHUTDOWN;
+ else
+ se_cmd->cmd_cnt = se_sess->cmd_cnt;
+ }
if (ret && ack_kref)
target_put_sess_cmd(se_cmd);
@@ -2993,7 +3034,7 @@ static void target_free_cmd_mem(struct se_cmd *cmd)
static void target_release_cmd_kref(struct kref *kref)
{
struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
- struct se_session *se_sess = se_cmd->se_sess;
+ struct target_cmd_counter *cmd_cnt = se_cmd->cmd_cnt;
struct completion *free_compl = se_cmd->free_compl;
struct completion *abrt_compl = se_cmd->abrt_compl;
@@ -3004,7 +3045,8 @@ static void target_release_cmd_kref(struct kref *kref)
if (abrt_compl)
complete(abrt_compl);
- percpu_ref_put(&se_sess->cmd_count);
+ if (cmd_cnt)
+ percpu_ref_put(&cmd_cnt->refcnt);
}
/**
@@ -3123,46 +3165,65 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd)
}
EXPORT_SYMBOL(target_show_cmd);
-static void target_stop_session_confirm(struct percpu_ref *ref)
+static void target_stop_cmd_counter_confirm(struct percpu_ref *ref)
{
- struct se_session *se_sess = container_of(ref, struct se_session,
- cmd_count);
- complete_all(&se_sess->stop_done);
+ struct target_cmd_counter *cmd_cnt = container_of(ref,
+ struct target_cmd_counter,
+ refcnt);
+ complete_all(&cmd_cnt->stop_done);
+}
+
+/**
+ * target_stop_cmd_counter - Stop new IO from being added to the counter.
+ * @cmd_cnt: counter to stop
+ */
+static void target_stop_cmd_counter(struct target_cmd_counter *cmd_cnt)
+{
+ pr_debug("Stopping command counter.\n");
+ if (!atomic_cmpxchg(&cmd_cnt->stopped, 0, 1))
+ percpu_ref_kill_and_confirm(&cmd_cnt->refcnt,
+ target_stop_cmd_counter_confirm);
}
/**
* target_stop_session - Stop new IO from being queued on the session.
- * @se_sess: session to stop
+ * @se_sess: session to stop
*/
void target_stop_session(struct se_session *se_sess)
{
- pr_debug("Stopping session queue.\n");
- if (atomic_cmpxchg(&se_sess->stopped, 0, 1) == 0)
- percpu_ref_kill_and_confirm(&se_sess->cmd_count,
- target_stop_session_confirm);
+ target_stop_cmd_counter(se_sess->cmd_cnt);
}
EXPORT_SYMBOL(target_stop_session);
/**
- * target_wait_for_sess_cmds - Wait for outstanding commands
- * @se_sess: session to wait for active I/O
+ * target_wait_for_cmds - Wait for outstanding cmds.
+ * @cmd_cnt: counter to wait for active I/O for.
*/
-void target_wait_for_sess_cmds(struct se_session *se_sess)
+static void target_wait_for_cmds(struct target_cmd_counter *cmd_cnt)
{
int ret;
- WARN_ON_ONCE(!atomic_read(&se_sess->stopped));
+ WARN_ON_ONCE(!atomic_read(&cmd_cnt->stopped));
do {
pr_debug("Waiting for running cmds to complete.\n");
- ret = wait_event_timeout(se_sess->cmd_count_wq,
- percpu_ref_is_zero(&se_sess->cmd_count),
- 180 * HZ);
+ ret = wait_event_timeout(cmd_cnt->refcnt_wq,
+ percpu_ref_is_zero(&cmd_cnt->refcnt),
+ 180 * HZ);
} while (ret <= 0);
- wait_for_completion(&se_sess->stop_done);
+ wait_for_completion(&cmd_cnt->stop_done);
pr_debug("Waiting for cmds done.\n");
}
+
+/**
+ * target_wait_for_sess_cmds - Wait for outstanding commands
+ * @se_sess: session to wait for active I/O
+ */
+void target_wait_for_sess_cmds(struct se_session *se_sess)
+{
+ target_wait_for_cmds(se_sess->cmd_cnt);
+}
EXPORT_SYMBOL(target_wait_for_sess_cmds);
/*
@@ -600,6 +600,7 @@ struct iscsit_conn {
struct iscsi_tpg_np *tpg_np;
/* Pointer to parent session */
struct iscsit_session *sess;
+ struct target_cmd_counter *cmd_cnt;
int bitmap_id;
int rx_thread_active;
struct task_struct *rx_thread;
@@ -494,6 +494,7 @@ struct se_cmd {
struct se_lun *se_lun;
/* Only used for internal passthrough and legacy TCM fabric modules */
struct se_session *se_sess;
+ struct target_cmd_counter *cmd_cnt;
struct se_tmr_req *se_tmr_req;
struct llist_node se_cmd_list;
struct completion *free_compl;
@@ -619,22 +620,26 @@ static inline struct se_node_acl *fabric_stat_to_nacl(struct config_item *item)
acl_fabric_stat_group);
}
-struct se_session {
+struct target_cmd_counter {
+ struct percpu_ref refcnt;
+ wait_queue_head_t refcnt_wq;
+ struct completion stop_done;
atomic_t stopped;
+};
+
+struct se_session {
u64 sess_bin_isid;
enum target_prot_op sup_prot_ops;
enum target_prot_type sess_prot_type;
struct se_node_acl *se_node_acl;
struct se_portal_group *se_tpg;
void *fabric_sess_ptr;
- struct percpu_ref cmd_count;
struct list_head sess_list;
struct list_head sess_acl_list;
spinlock_t sess_cmd_lock;
- wait_queue_head_t cmd_count_wq;
- struct completion stop_done;
void *sess_cmd_map;
struct sbitmap_queue sess_tag_pool;
+ struct target_cmd_counter *cmd_cnt;
};
struct se_device;
iSCSI needs to wait on outstanding commands like how srp and the FC/fcoe drivers do. It can't use target_stop_session because for MCS support we can't stop the entire session during recovery because if other connections are ok then we want to be able to continue to execute IO on them. This patch moves the per session cmd counters to a new struct, so iSCSI can allocate it per connection. The xcopy code can also just not allocate it in the future since it doesn't need to track commands. Signed-off-by: Mike Christie <michael.christie@oracle.com> --- drivers/target/target_core_tpg.c | 2 +- drivers/target/target_core_transport.c | 141 ++++++++++++++++------- include/target/iscsi/iscsi_target_core.h | 1 + include/target/target_core_base.h | 13 ++- 4 files changed, 112 insertions(+), 45 deletions(-)