[15/15] runjob: customize the loose-objects batch size

Commit Message

Linus Arver via GitGitGadget April 3, 2020, 8:48 p.m. UTC

From: Derrick Stolee <[e-mail]>

Allow a user to override the default number of loose objects to
place into a new pack-file as part of the loose-objects job. This
can be done via the job.loose-objects.batchSize config option or
the --batch-size=<count> option in the 'git run-job' command. The
config value is checked once per run of 'git run-job loose-objects'
so an instance started by 'git job-runner' will use new values
automatically without restarting the 'git job-runner' process.

Signed-off-by: Derrick Stolee <[e-mail]>
---
 Documentation/config/job.txt  |  6 ++++++
 Documentation/git-run-job.txt |  8 +++++---
 builtin/run-job.c             | 31 ++++++++++++++++++++++++++-----
 3 files changed, 37 insertions(+), 8 deletions(-)

Patch

diff --git a/Documentation/config/job.txt b/Documentation/config/job.txt
index 6c22a40dd36..baa5b927e14 100644
--- a/Documentation/config/job.txt
+++ b/Documentation/config/job.txt
@@ -16,6 +16,12 @@  job.<job-name>.lastRun::
 	can manually update this to a later time to delay a specific
 	job on this repository.
 
+job.loose-objects.batchSize::
+	This string value `<count>` limits the number of loose-objects
+	collected into a single pack-file during the `loose-objects`
+	job. Default batch size is fifty thousand. See linkgit:git-run-job[1]
+	for more details.
+
 job.pack-files.batchSize::
 	This string value `<size>` will be passed to the
 	`git multi-pack-index repack --batch-size=<size>` command as
diff --git a/Documentation/git-run-job.txt b/Documentation/git-run-job.txt
index c6d5674d699..73210791533 100644
--- a/Documentation/git-run-job.txt
+++ b/Documentation/git-run-job.txt
@@ -67,9 +67,11 @@  commands, it follows a two-step process. First, it deletes any loose
 objects that already exist in a pack-file; concurrent Git processes will
 examine the pack-file for the object data instead of the loose object.
 Second, it creates a new pack-file (starting with "loose-") containing
-a batch of loose objects. The batch size is limited to 50 thousand
-objects to prevent the job from taking too long on a repository with
-many loose objects.
+a batch of loose objects.
++
+By default, the batch size is limited to 50 thousand objects to prevent
+the job from taking too long on a repository with many loose objects.
+This can be overridden with the `--batch-size=<count>` option.
 
 'pack-files'::
 
diff --git a/builtin/run-job.c b/builtin/run-job.c
index 76765535e09..b7c5a74cdbb 100644
--- a/builtin/run-job.c
+++ b/builtin/run-job.c
@@ -13,6 +13,11 @@  static char const * const builtin_run_job_usage[] = {
 	NULL
 };
 
+static char const * const builtin_run_job_loose_objects_usage[] = {
+	N_("git run-job loose-objects [--batch-size=<count>]"),
+	NULL
+};
+
 static char const * const builtin_run_job_pack_file_usage[] = {
 	N_("git run-job pack-files [--batch-size=<size>]"),
 	NULL
@@ -183,7 +188,7 @@  static int write_loose_object_to_stdin(const struct object_id *oid,
 	return ++(d->count) > d->batch_size;
 }
 
-static int pack_loose(void)
+static int pack_loose(int batch_size)
 {
 	int result = 0;
 	struct write_loose_object_data data;
@@ -219,7 +224,7 @@  static int pack_loose(void)
 
 	data.in = xfdopen(pack_proc->in, "w");
 	data.count = 0;
-	data.batch_size = 50000;
+	data.batch_size = batch_size;
 
 	for_each_loose_file_in_objdir(the_repository->objects->odb->path,
 				      write_loose_object_to_stdin,
@@ -240,9 +245,25 @@  static int pack_loose(void)
 	return result;
 }
 
-static int run_loose_objects_job(void)
+static int run_loose_objects_job(int argc, const char **argv)
 {
-	return prune_packed() || pack_loose();
+	static int batch_size;
+	static struct option builtin_run_job_loose_objects_options[] = {
+		OPT_INTEGER(0, "batch-size", &batch_size,
+			    N_("specify the maximum number of loose objects to store in a pack-file")),
+		OPT_END(),
+	};
+
+	if (repo_config_get_int(the_repository,
+				"job.loose-objects.batchsize",
+				&batch_size))
+		batch_size = 50000;
+
+	argc = parse_options(argc, argv, NULL,
+			     builtin_run_job_loose_objects_options,
+			     builtin_run_job_loose_objects_usage, 0);
+
+	return prune_packed() || pack_loose(batch_size);
 }
 
 static int multi_pack_index_write(void)
@@ -427,7 +448,7 @@  int cmd_run_job(int argc, const char **argv, const char *prefix)
 		if (!strcmp(argv[0], "fetch"))
 			return run_fetch_job();
 		if (!strcmp(argv[0], "loose-objects"))
-			return run_loose_objects_job();
+			return run_loose_objects_job(argc, argv);
 		if (!strcmp(argv[0], "pack-files"))
 			return run_pack_files_job(argc, argv);
 	}