[v2,10/11] maintenance: add auto condition for commit-graph task

Commit Message

Philippe Blain via GitGitGadget Aug. 18, 2020, 2:23 p.m. UTC

From: Derrick Stolee <[e-mail]>

Instead of writing a new commit-graph in every 'git maintenance run
--auto' process (when maintenance.commit-graph.enalbed is configured to
be true), only write when there are "enough" commits not in a
commit-graph file.

This count is controlled by the maintenance.commit-graph.auto config
option.

To compute the count, use a depth-first search starting at each ref, and
leaving markers using the PARENT1 flag. If this count reaches the limit,
then terminate early and start the task. Otherwise, this operation will
peel every ref and parse the commit it points to. If these are all in
the commit-graph, then this is typically a very fast operation. Users
with many refs might feel a slow-down, and hence could consider updating
their limit to be very small. A negative value will force the step to
run every time.

Signed-off-by: Derrick Stolee <[e-mail]>
---
 Documentation/config/maintenance.txt | 10 ++++
 builtin/gc.c                         | 76 ++++++++++++++++++++++++++++
 object.h                             |  1 +
 3 files changed, 87 insertions(+)

Comments

Jonathan Tan Aug. 19, 2020, 12:09 a.m. UTC | #1

> Instead of writing a new commit-graph in every 'git maintenance run
> --auto' process (when maintenance.commit-graph.enalbed is configured to

s/enalbed/enabled/

> +/* Remember to update object flag allocation in object.h */
> +#define PARENT1		(1u<<16)

Why this name? "SEEN" seems perfectly fine.

> +static int num_commits_not_in_graph = 0;
> +static int limit_commits_not_in_graph = 100;
> +
> +static int dfs_on_ref(const char *refname,
> +		      const struct object_id *oid, int flags,
> +		      void *cb_data)
> +{

[snip]

> +static int should_write_commit_graph(void)
> +{
> +	int result;
> +
> +	git_config_get_int("maintenance.commit-graph.auto",
> +			   &limit_commits_not_in_graph);
> +
> +	if (!limit_commits_not_in_graph)
> +		return 0;
> +	if (limit_commits_not_in_graph < 0)
> +		return 1;
> +
> +	result = for_each_ref(dfs_on_ref, NULL);

I don't like introducing the mutable global num_commits_not_in_graph
especially when there seems to be at least one easy alternative (e.g.
putting it in cb_data) but I know that this is a pattern than existing
code.

Derrick Stolee Aug. 19, 2020, 3:15 p.m. UTC | #2

On 8/18/2020 8:09 PM, Jonathan Tan wrote:
>> Instead of writing a new commit-graph in every 'git maintenance run
>> --auto' process (when maintenance.commit-graph.enalbed is configured to
> 
> s/enalbed/enabled/
> 
>> +/* Remember to update object flag allocation in object.h */
>> +#define PARENT1		(1u<<16)
> 
> Why this name? "SEEN" seems perfectly fine.

For some reason I thought that there might be issues using
SEEN (1u<<0) but trying it locally does not seem to be a
problem. I think I've just been bit by how revision.c uses
it a bit too often. The use here is independent enough to
not cause problems.

>> +static int num_commits_not_in_graph = 0;
>> +static int limit_commits_not_in_graph = 100;
>> +
>> +static int dfs_on_ref(const char *refname,
>> +		      const struct object_id *oid, int flags,
>> +		      void *cb_data)
>> +{
> 
> [snip]
> 
>> +static int should_write_commit_graph(void)
>> +{
>> +	int result;
>> +
>> +	git_config_get_int("maintenance.commit-graph.auto",
>> +			   &limit_commits_not_in_graph);
>> +
>> +	if (!limit_commits_not_in_graph)
>> +		return 0;
>> +	if (limit_commits_not_in_graph < 0)
>> +		return 1;
>> +
>> +	result = for_each_ref(dfs_on_ref, NULL);
> 
> I don't like introducing the mutable global num_commits_not_in_graph
> especially when there seems to be at least one easy alternative (e.g.
> putting it in cb_data) but I know that this is a pattern than existing
> code.

I should have done this right in the first place. Thanks for
catching it.

-Stolee

Patch

diff --git a/Documentation/config/maintenance.txt b/Documentation/config/maintenance.txt
index 370cbfb42f..9bd69b9df3 100644
--- a/Documentation/config/maintenance.txt
+++ b/Documentation/config/maintenance.txt
@@ -2,3 +2,13 @@  maintenance.<task>.enabled::
 	This boolean config option controls whether the maintenance task
 	with name `<task>` is run when no `--task` option is specified.
 	By default, only `maintenance.gc.enabled` is true.
+
+maintenance.commit-graph.auto::
+	This integer config option controls how often the `commit-graph` task
+	should be run as part of `git maintenance run --auto`. If zero, then
+	the `commit-graph` task will not run with the `--auto` option. A
+	negative value will force the task to run every time. Otherwise, a
+	positive value implies the command should run when the number of
+	reachable commits that are not in the commit-graph file is at least
+	the value of `maintenance.commit-graph.auto`. The default value is
+	100.
diff --git a/builtin/gc.c b/builtin/gc.c
index c54c3070b8..73e485bfc0 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -28,6 +28,7 @@ 
 #include "blob.h"
 #include "tree.h"
 #include "promisor-remote.h"
+#include "refs.h"
 
 #define FAILED_RUN "failed to run %s"
 
@@ -710,6 +711,80 @@  struct maintenance_opts {
 	int quiet;
 };
 
+/* Remember to update object flag allocation in object.h */
+#define PARENT1		(1u<<16)
+
+static int num_commits_not_in_graph = 0;
+static int limit_commits_not_in_graph = 100;
+
+static int dfs_on_ref(const char *refname,
+		      const struct object_id *oid, int flags,
+		      void *cb_data)
+{
+	int result = 0;
+	struct object_id peeled;
+	struct commit_list *stack = NULL;
+	struct commit *commit;
+
+	if (!peel_ref(refname, &peeled))
+		oid = &peeled;
+	if (oid_object_info(the_repository, oid, NULL) != OBJ_COMMIT)
+		return 0;
+
+	commit = lookup_commit(the_repository, oid);
+	if (!commit)
+		return 0;
+	if (parse_commit(commit))
+		return 0;
+
+	commit_list_append(commit, &stack);
+
+	while (!result && stack) {
+		struct commit_list *parent;
+
+		commit = pop_commit(&stack);
+
+		for (parent = commit->parents; parent; parent = parent->next) {
+			if (parse_commit(parent->item) ||
+			    commit_graph_position(parent->item) != COMMIT_NOT_FROM_GRAPH ||
+			    parent->item->object.flags & PARENT1)
+				continue;
+
+			parent->item->object.flags |= PARENT1;
+			num_commits_not_in_graph++;
+
+			if (num_commits_not_in_graph >= limit_commits_not_in_graph) {
+				result = 1;
+				break;
+			}
+
+			commit_list_append(parent->item, &stack);
+		}
+	}
+
+	free_commit_list(stack);
+	return result;
+}
+
+static int should_write_commit_graph(void)
+{
+	int result;
+
+	git_config_get_int("maintenance.commit-graph.auto",
+			   &limit_commits_not_in_graph);
+
+	if (!limit_commits_not_in_graph)
+		return 0;
+	if (limit_commits_not_in_graph < 0)
+		return 1;
+
+	result = for_each_ref(dfs_on_ref, NULL);
+
+	clear_commit_marks_all(PARENT1);
+
+	return result;
+}
+
 static int run_write_commit_graph(struct maintenance_opts *opts)
 {
 	struct child_process child = CHILD_PROCESS_INIT;
@@ -822,6 +897,7 @@  static struct maintenance_task tasks[] = {
 	[TASK_COMMIT_GRAPH] = {
 		"commit-graph",
 		maintenance_task_commit_graph,
+		should_write_commit_graph,
 	},
 };
 
diff --git a/object.h b/object.h
index 96a2105859..30d4e0f6a0 100644
--- a/object.h
+++ b/object.h
@@ -74,6 +74,7 @@  struct object_array {
  * list-objects-filter.c:                                      21
  * builtin/fsck.c:           0--3
  * builtin/index-pack.c:                                     2021
+ * builtin/maintenance.c:                           16
  * builtin/pack-objects.c:                                   20
  * builtin/reflog.c:                   10--12
  * builtin/show-branch.c:    0-------------------------------------------26