@@ -87,6 +87,21 @@ loose-objects::
thousand objects to prevent the job from taking too long on a
repository with many loose objects.
+incremental-repack::
+ The `incremental-repack` job repacks the object directory
+ using the `multi-pack-index` feature. In order to prevent race
+ conditions with concurrent Git commands, it follows a two-step
+ process. First, it deletes any pack-files included in the
+ `multi-pack-index` where none of the objects in the
+ `multi-pack-index` reference those pack-files; this only happens
+ if all objects in the pack-file are also stored in a newer
+ pack-file. Second, it selects a group of pack-files whose "expected
+ size" is below the batch size until the group has total expected
+ size at least the batch size; see the `--batch-size` option for
+ the `repack` subcommand in linkgit:git-multi-pack-index[1]. The
+ default batch-size is zero, which is a special case that attempts
+ to repack all pack-files into a single pack-file.
+
OPTIONS
-------
--auto::
@@ -29,6 +29,7 @@
#include "tree.h"
#include "promisor-remote.h"
#include "remote.h"
+#include "midx.h"
#define FAILED_RUN "failed to run %s"
@@ -924,6 +925,112 @@ static int maintenance_task_loose_objects(void)
return prune_packed() || pack_loose();
}
+static int multi_pack_index_write(void)
+{
+ struct child_process child = CHILD_PROCESS_INIT;
+
+ child.git_cmd = 1;
+ strvec_pushl(&child.args, "multi-pack-index", "write", NULL);
+
+ if (opts.quiet)
+ strvec_push(&child.args, "--no-progress");
+
+ if (run_command(&child))
+ return error(_("failed to write multi-pack-index"));
+
+ return 0;
+}
+
+static int rewrite_multi_pack_index(void)
+{
+ struct repository *r = the_repository;
+ char *midx_name = get_midx_filename(r->objects->odb->path);
+
+ unlink(midx_name);
+ free(midx_name);
+
+ return multi_pack_index_write();
+}
+
+static int multi_pack_index_verify(const char *message)
+{
+ struct child_process child = CHILD_PROCESS_INIT;
+
+ child.git_cmd = 1;
+ strvec_pushl(&child.args, "multi-pack-index", "verify", NULL);
+
+ if (opts.quiet)
+ strvec_push(&child.args, "--no-progress");
+
+ if (run_command(&child)) {
+ warning(_("'git multi-pack-index verify' failed %s"), message);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int multi_pack_index_expire(void)
+{
+ struct child_process child = CHILD_PROCESS_INIT;
+
+ child.git_cmd = 1;
+ strvec_pushl(&child.args, "multi-pack-index", "expire", NULL);
+
+ if (opts.quiet)
+ strvec_push(&child.args, "--no-progress");
+
+ close_object_store(the_repository->objects);
+
+ if (run_command(&child))
+ return error(_("'git multi-pack-index expire' failed"));
+
+ return 0;
+}
+
+static int multi_pack_index_repack(void)
+{
+ struct child_process child = CHILD_PROCESS_INIT;
+
+ child.git_cmd = 1;
+ strvec_pushl(&child.args, "multi-pack-index", "repack", NULL);
+
+ if (opts.quiet)
+ strvec_push(&child.args, "--no-progress");
+
+ strvec_push(&child.args, "--batch-size=0");
+
+ close_object_store(the_repository->objects);
+
+ if (run_command(&child))
+ return error(_("'git multi-pack-index repack' failed"));
+
+ return 0;
+}
+
+static int maintenance_task_incremental_repack(void)
+{
+ prepare_repo_settings(the_repository);
+ if (!the_repository->settings.core_multi_pack_index) {
+ warning(_("skipping incremental-repack task because core.multiPackIndex is disabled"));
+ return 0;
+ }
+
+ if (multi_pack_index_write())
+ return 1;
+ if (multi_pack_index_verify("after initial write"))
+ return rewrite_multi_pack_index();
+ if (multi_pack_index_expire())
+ return 1;
+ if (multi_pack_index_verify("after expire step"))
+ return !!rewrite_multi_pack_index();
+ if (multi_pack_index_repack())
+ return 1;
+ if (multi_pack_index_verify("after repack step"))
+ return !!rewrite_multi_pack_index();
+ return 0;
+}
+
typedef int maintenance_task_fn(void);
struct maintenance_task {
@@ -937,6 +1044,7 @@ struct maintenance_task {
enum maintenance_task_label {
TASK_PREFETCH,
TASK_LOOSE_OBJECTS,
+ TASK_INCREMENTAL_REPACK,
TASK_GC,
TASK_COMMIT_GRAPH,
@@ -953,6 +1061,10 @@ static struct maintenance_task tasks[] = {
"loose-objects",
maintenance_task_loose_objects,
},
+ [TASK_INCREMENTAL_REPACK] = {
+ "incremental-repack",
+ maintenance_task_incremental_repack,
+ },
[TASK_GC] = {
"gc",
maintenance_task_gc,
@@ -37,7 +37,7 @@
#define PACK_EXPIRED UINT_MAX
-static char *get_midx_filename(const char *object_dir)
+char *get_midx_filename(const char *object_dir)
{
return xstrfmt("%s/pack/multi-pack-index", object_dir);
}
@@ -37,6 +37,7 @@ struct multi_pack_index {
#define MIDX_PROGRESS (1 << 0)
+char *get_midx_filename(const char *object_dir);
struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local);
int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id);
int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result);
@@ -3,6 +3,7 @@
test_description='multi-pack-indexes'
. ./test-lib.sh
+GIT_TEST_MULTI_PACK_INDEX=0
objdir=.git/objects
midx_read_expect () {
@@ -5,6 +5,7 @@ test_description='git maintenance builtin'
. ./test-lib.sh
GIT_TEST_COMMIT_GRAPH=0
+GIT_TEST_MULTI_PACK_INDEX=0
test_expect_success 'help text' '
test_expect_code 129 git maintenance -h 2>err &&
@@ -106,4 +107,41 @@ test_expect_success 'loose-objects task' '
test_cmp packs-between packs-after
'
+test_expect_success 'incremental-repack task' '
+ packDir=.git/objects/pack &&
+ for i in $(test_seq 1 5)
+ do
+ test_commit $i || return 1
+ done &&
+
+ # Create three disjoint pack-files with size BIG, small, small.
+ echo HEAD~2 | git pack-objects --revs $packDir/test-1 &&
+ test_tick &&
+ git pack-objects --revs $packDir/test-2 <<-\EOF &&
+ HEAD~1
+ ^HEAD~2
+ EOF
+ test_tick &&
+ git pack-objects --revs $packDir/test-3 <<-\EOF &&
+ HEAD
+ ^HEAD~1
+ EOF
+ rm -f $packDir/pack-* &&
+ rm -f $packDir/loose-* &&
+ ls $packDir/*.pack >packs-before &&
+ test_line_count = 3 packs-before &&
+
+ # the job repacks the two into a new pack, but does not
+ # delete the old ones.
+ git maintenance run --task=incremental-repack &&
+ ls $packDir/*.pack >packs-between &&
+ test_line_count = 4 packs-between &&
+
+ # the job deletes the two old packs, and does not write
+ # a new one because only one pack remains.
+ git maintenance run --task=incremental-repack &&
+ ls .git/objects/pack/*.pack >packs-after &&
+ test_line_count = 1 packs-after
+'
+
test_done