[v2,3/9] trace2: defer free of thread local storage until program exit.

Commit Message

Jeff Hostetler Dec. 28, 2021, 7:36 p.m. UTC

From: Jeff Hostetler <[e-mail]>

Defer freeing of the Trace2 per-thread thread local storage until
program exit.  Create a global list to own them.

Trace2 thread local storage data is allocated when a thread is created
and associated with that thread.  Previously, that storage was deleted
when the thread exited.  Now at thread exit, we simply disassociate
the data from the thread and let the global list manage the cleanup.

This will be used by a later commit when we add "counters" and
stopwatch-style "timers" to the Trace2 API.  We will add those fields
to the thread local storage block and allow each thread to efficiently
(without locks) accumulate counter and timer data.  At program exit,
the main thread will run thru the global list and compute and report
totals before freeing the list.

Signed-off-by: Jeff Hostetler <[e-mail]>
---
 trace2/tr2_tls.c | 38 +++++++++++++++++++++++++++++++-------
 trace2/tr2_tls.h |  3 ++-
 2 files changed, 33 insertions(+), 8 deletions(-)

Patch

diff --git a/trace2/tr2_tls.c b/trace2/tr2_tls.c
index ed99a234b95..78538d5e522 100644
--- a/trace2/tr2_tls.c
+++ b/trace2/tr2_tls.c
@@ -15,6 +15,18 @@  static uint64_t tr2tls_us_start_process;
 static pthread_mutex_t tr2tls_mutex;
 static pthread_key_t tr2tls_key;
 
+/*
+ * This list owns all of the thread-specific CTX data.
+ *
+ * While a thread is alive it is associated with a CTX (owned by this
+ * list) and that CTX is installed in the thread's TLS data area.
+ * When a thread exits, it is disassociated from its CTX, but the (now
+ * dormant) CTX is held in this list until program exit.
+ *
+ * Similarly, `tr2tls_thread_main` points to a CTX contained within
+ * this list.
+ */
+static struct tr2tls_thread_ctx *tr2tls_ctx_list; /* modify under lock */
 static int tr2_next_thread_id; /* modify under lock */
 
 void tr2tls_start_process_clock(void)
@@ -56,6 +68,14 @@  struct tr2tls_thread_ctx *tr2tls_create_self(const char *thread_name,
 	ctx->array_us_start = (uint64_t *)xcalloc(ctx->alloc, sizeof(uint64_t));
 	ctx->array_us_start[ctx->nr_open_regions++] = us_thread_start;
 
+	/*
+	 * Link this CTX into the CTX list and make it the head.
+	 */
+	pthread_mutex_lock(&tr2tls_mutex);
+	ctx->next_ctx = tr2tls_ctx_list;
+	tr2tls_ctx_list = ctx;
+	pthread_mutex_unlock(&tr2tls_mutex);
+
 	pthread_setspecific(tr2tls_key, ctx);
 
 	return ctx;
@@ -91,14 +111,7 @@  int tr2tls_is_main_thread(void)
 
 void tr2tls_unset_self(void)
 {
-	struct tr2tls_thread_ctx *ctx;
-
-	ctx = tr2tls_get_self();
-
 	pthread_setspecific(tr2tls_key, NULL);
-
-	free(ctx->array_us_start);
-	free(ctx);
 }
 
 void tr2tls_push_self(uint64_t us_now)
@@ -162,11 +175,22 @@  void tr2tls_init(void)
 
 void tr2tls_release(void)
 {
+	struct tr2tls_thread_ctx *ctx = tr2tls_ctx_list;
+
 	tr2tls_unset_self();
 	tr2tls_thread_main = NULL;
 
 	pthread_mutex_destroy(&tr2tls_mutex);
 	pthread_key_delete(tr2tls_key);
+
+	while (ctx) {
+		struct tr2tls_thread_ctx *next = ctx->next_ctx;
+
+		free(ctx->array_us_start);
+		free(ctx);
+
+		ctx = next;
+	}
 }
 
 int tr2tls_locked_increment(int *p)
diff --git a/trace2/tr2_tls.h b/trace2/tr2_tls.h
index 64d97c5ac03..889010ec1ff 100644
--- a/trace2/tr2_tls.h
+++ b/trace2/tr2_tls.h
@@ -4,6 +4,7 @@ 
 #include "strbuf.h"
 
 struct tr2tls_thread_ctx {
+	struct tr2tls_thread_ctx *next_ctx;
 	uint64_t *array_us_start;
 	size_t alloc;
 	size_t nr_open_regions; /* plays role of "nr" in ALLOC_GROW */
@@ -37,7 +38,7 @@  struct tr2tls_thread_ctx *tr2tls_get_self(void);
 int tr2tls_is_main_thread(void);
 
 /*
- * Free our TLS data.
+ * Disassociate thread's TLS CTX data from the thread.
  */
 void tr2tls_unset_self(void);