[04/23] t/helper/fsmonitor-client: create stress test

Commit Message

Jeff Hostetler Feb. 15, 2022, 3:59 p.m. UTC

From: Jeff Hostetler <[e-mail]>

Create a stress test to hammer on the fsmonitor daemon.
Create a client-side thread pool of n threads and have
each of them make m requests as fast as they can.

NEEDSWORK: This is just the client-side thread pool and
is useful for interactive testing and experimentation.
We need to add a script test to drive this.

Signed-off-by: Jeff Hostetler <[e-mail]>
---
 t/helper/test-fsmonitor-client.c | 105 +++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)

Comments

Derrick Stolee Feb. 24, 2022, 2:58 p.m. UTC | #1

On 2/15/2022 10:59 AM, Jeff Hostetler via GitGitGadget wrote:
> From: Jeff Hostetler <[e-mail]>
> 
> Create a stress test to hammer on the fsmonitor daemon.
> Create a client-side thread pool of n threads and have
> each of them make m requests as fast as they can.
> 
> NEEDSWORK: This is just the client-side thread pool and
> is useful for interactive testing and experimentation.
> We need to add a script test to drive this.

I haven't gotten far enough in the series to know if you
_do_ use this in a test eventually. If so, this NEEDSWORK
could be replaced with a mention of a future change.

> +	/*
> +	 * TODO Decide if/when to return an error or call die().
> +	 */
> +	return 0;

This TODO could be cleaned up.

Thanks,
-Stolee

Jeff Hostetler March 1, 2022, 7:37 p.m. UTC | #2

On 2/24/22 9:58 AM, Derrick Stolee wrote:
> On 2/15/2022 10:59 AM, Jeff Hostetler via GitGitGadget wrote:
>> From: Jeff Hostetler <[e-mail]>
>>
>> Create a stress test to hammer on the fsmonitor daemon.
>> Create a client-side thread pool of n threads and have
>> each of them make m requests as fast as they can.
>>
>> NEEDSWORK: This is just the client-side thread pool and
>> is useful for interactive testing and experimentation.
>> We need to add a script test to drive this.
> 
> I haven't gotten far enough in the series to know if you
> _do_ use this in a test eventually. If so, this NEEDSWORK
> could be replaced with a mention of a future change.

Right. I mainly use this test helper to hammer on the
daemon during interactive tests.  I don't have a script
to actually use.  I'm currently not sure what that would
look like given our test script framework.

> 
>> +	/*
>> +	 * TODO Decide if/when to return an error or call die().
>> +	 */
>> +	return 0;
> 
> This TODO could be cleaned up.

good catch.  thanks!

Jeff

Patch

diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c
index f7a5b3a32fa..9dd2f9af553 100644
--- a/t/helper/test-fsmonitor-client.c
+++ b/t/helper/test-fsmonitor-client.c
@@ -7,6 +7,8 @@ 
 #include "cache.h"
 #include "parse-options.h"
 #include "fsmonitor-ipc.h"
+#include "thread-utils.h"
+#include "trace2.h"
 
 #ifndef HAVE_FSMONITOR_DAEMON_BACKEND
 int cmd__fsmonitor_client(int argc, const char **argv)
@@ -79,20 +81,120 @@  static int do_send_flush(void)
 	return 0;
 }
 
+struct hammer_thread_data
+{
+	pthread_t pthread_id;
+	int thread_nr;
+
+	int nr_requests;
+	const char *token;
+
+	int sum_successful;
+	int sum_errors;
+};
+
+static void *hammer_thread_proc(void *_hammer_thread_data)
+{
+	struct hammer_thread_data *data = _hammer_thread_data;
+	struct strbuf answer = STRBUF_INIT;
+	int k;
+	int ret;
+
+	trace2_thread_start("hammer");
+
+	for (k = 0; k < data->nr_requests; k++) {
+		strbuf_reset(&answer);
+
+		ret = fsmonitor_ipc__send_query(data->token, &answer);
+		if (ret < 0)
+			data->sum_errors++;
+		else
+			data->sum_successful++;
+	}
+
+	strbuf_release(&answer);
+	trace2_thread_exit();
+	return NULL;
+}
+
+/*
+ * Start a pool of client threads that will each send a series of
+ * commands to the daemon.
+ *
+ * The goal is to overload the daemon with a sustained series of
+ * concurrent requests.
+ */
+static int do_hammer(const char *token, int nr_threads, int nr_requests)
+{
+	struct hammer_thread_data *data = NULL;
+	int k;
+	int sum_join_errors = 0;
+	int sum_commands = 0;
+	int sum_errors = 0;
+
+	if (!token || !*token)
+		token = get_token_from_index();
+	if (nr_threads < 1)
+		nr_threads = 1;
+	if (nr_requests < 1)
+		nr_requests = 1;
+
+	CALLOC_ARRAY(data, nr_threads);
+
+	for (k = 0; k < nr_threads; k++) {
+		struct hammer_thread_data *p = &data[k];
+		p->thread_nr = k;
+		p->nr_requests = nr_requests;
+		p->token = token;
+
+		if (pthread_create(&p->pthread_id, NULL, hammer_thread_proc, p)) {
+			warning("failed to create thread[%d] skipping remainder", k);
+			nr_threads = k;
+			break;
+		}
+	}
+
+	for (k = 0; k < nr_threads; k++) {
+		struct hammer_thread_data *p = &data[k];
+
+		if (pthread_join(p->pthread_id, NULL))
+			sum_join_errors++;
+		sum_commands += p->sum_successful;
+		sum_errors += p->sum_errors;
+	}
+
+	fprintf(stderr, "HAMMER: [threads %d][requests %d] [ok %d][err %d][join %d]\n",
+		nr_threads, nr_requests, sum_commands, sum_errors, sum_join_errors);
+
+	free(data);
+
+	/*
+	 * TODO Decide if/when to return an error or call die().
+	 */
+	return 0;
+}
+
 int cmd__fsmonitor_client(int argc, const char **argv)
 {
 	const char *subcmd;
 	const char *token = NULL;
+	int nr_threads = 1;
+	int nr_requests = 1;
 
 	const char * const fsmonitor_client_usage[] = {
 		N_("test-helper fsmonitor-client query [<token>]"),
 		N_("test-helper fsmonitor-client flush"),
+		N_("test-helper fsmonitor-client hammer [<token>] [<threads>] [<requests>]"),
 		NULL,
 	};
 
 	struct option options[] = {
 		OPT_STRING(0, "token", &token, N_("token"),
 			   N_("command token to send to the server")),
+
+		OPT_INTEGER(0, "threads", &nr_threads, N_("number of client threads")),
+		OPT_INTEGER(0, "requests", &nr_requests, N_("number of requests per thread")),
+
 		OPT_END()
 	};
 
@@ -116,6 +218,9 @@  int cmd__fsmonitor_client(int argc, const char **argv)
 	if (!strcmp(subcmd, "flush"))
 		return !!do_send_flush();
 
+	if (!strcmp(subcmd, "hammer"))
+		return !!do_hammer(token, nr_threads, nr_requests);
+
 	die("Unhandled subcommand: '%s'", subcmd);
 }
 #endif