@@ -1191,6 +1191,7 @@ static void write_reused_pack(struct bitmapped_pack *reuse_packfile,
size_t pos = (i * BITS_IN_EWORD);
for (offset = 0; offset < BITS_IN_EWORD; ++offset) {
+ uint32_t pack_pos;
if ((word >> offset) == 0)
break;
@@ -1199,14 +1200,41 @@ static void write_reused_pack(struct bitmapped_pack *reuse_packfile,
continue;
if (pos + offset >= reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr)
goto done;
- /*
- * Can use bit positions directly, even for MIDX
- * bitmaps. See comment in try_partial_reuse()
- * for why.
- */
- write_reused_pack_one(reuse_packfile->p,
- pos + offset - reuse_packfile->bitmap_pos,
- f, pack_start, &w_curs);
+
+ if (reuse_packfile->bitmap_pos) {
+ /*
+ * When doing multi-pack reuse on a
+ * non-preferred pack, translate bit positions
+ * from the MIDX pseudo-pack order back to their
+ * pack-relative positions before attempting
+ * reuse.
+ */
+ struct multi_pack_index *m = reuse_packfile->from_midx;
+ uint32_t midx_pos;
+ off_t pack_ofs;
+
+ if (!m)
+ BUG("non-zero bitmap position without MIDX");
+
+ midx_pos = pack_pos_to_midx(m, pos + offset);
+ pack_ofs = nth_midxed_offset(m, midx_pos);
+
+ if (offset_to_pack_pos(reuse_packfile->p,
+ pack_ofs, &pack_pos) < 0)
+ BUG("could not find expected object at offset %"PRIuMAX" in pack %s",
+ (uintmax_t)pack_ofs,
+ pack_basename(reuse_packfile->p));
+ } else {
+ /*
+ * Can use bit positions directly, even for MIDX
+ * bitmaps. See comment in try_partial_reuse()
+ * for why.
+ */
+ pack_pos = pos + offset;
+ }
+
+ write_reused_pack_one(reuse_packfile->p, pack_pos, f,
+ pack_start, &w_curs);
display_progress(progress_state, ++written);
}
}
@@ -69,7 +69,7 @@ test_expect_success 'multi-pack reuse is disabled by default' '
test_pack_objects_reused_all 3 1
'
-test_expect_failure 'feature.experimental implies multi-pack reuse' '
+test_expect_success 'feature.experimental implies multi-pack reuse' '
test_config feature.experimental true &&
test_pack_objects_reused_all 6 2
@@ -86,7 +86,7 @@ test_expect_success 'enable multi-pack reuse' '
git config pack.allowPackReuse multi
'
-test_expect_failure 'reuse all objects from subset of bitmapped packs' '
+test_expect_success 'reuse all objects from subset of bitmapped packs' '
test_commit C &&
git repack -d &&
@@ -100,7 +100,7 @@ test_expect_failure 'reuse all objects from subset of bitmapped packs' '
test_pack_objects_reused 6 2 <in
'
-test_expect_failure 'reuse all objects from all packs' '
+test_expect_success 'reuse all objects from all packs' '
test_pack_objects_reused_all 9 3
'
@@ -194,7 +194,7 @@ test_expect_success 'omit delta with uninteresting base (same pack)' '
test_pack_objects_reused 3 1 <in
'
-test_expect_failure 'omit delta from uninteresting base (cross pack)' '
+test_expect_success 'omit delta from uninteresting base (cross pack)' '
cat >in <<-EOF &&
$(git rev-parse $base)
^$(git rev-parse $delta)
@@ -236,4 +236,27 @@ test_expect_success 'non-omitted delta in MIDX preferred pack' '
test_pack_objects_reused_all $(wc -l <expect) 1
'
+test_expect_success 'duplicate objects' '
+ git init duplicate-objects &&
+ (
+ cd duplicate-objects &&
+
+ git config pack.allowPackReuse multi &&
+
+ test_commit base &&
+
+ git repack -a &&
+
+ git rev-parse HEAD^{tree} >in &&
+ p="$(git pack-objects $packdir/pack <in)" &&
+
+ git multi-pack-index write --bitmap --preferred-pack=pack-$p.idx &&
+
+ objects_nr="$(git rev-list --count --all --objects)" &&
+ packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" &&
+
+ test_pack_objects_reused_all $objects_nr $packs_nr
+ )
+'
+
test_done
When reusing chunks verbatim from an existing source pack, the function write_reused_pack() first attempts to reuse whole words (via the function `write_reused_pack_verbatim()`), and then individual bits (via `write_reused_pack_one()`). In the non-MIDX case, all of this code works fine. Likewise, in the MIDX case, processing bits individually from the first (preferred) pack works fine. However, processing subsequent packs in the MIDX case is broken when there are duplicate objects among the set of MIDX'd packs. This is because we treat the individual bit positions as valid pack positions within the source pack(s), which does not account for gaps in the source pack, like we see when the MIDX must break ties between duplicate objects which appear in multiple packs. The broken code looks like: for (; i < reuse_packfile_bitmap->word_alloc; i++) { for (offset = 0; offset < BITS_IN_EWORD, offset++) { /* ... */ write_reused_pack_one(reuse_packfile->p, pos + offset - reuse_packfile->bitmap_pos, f, pack_start, &w_curs); } } , where the second argument is incorrect and does not account for gaps. Instead, make sure that we translate bit positions in the MIDX's pseudo-pack order to pack positions in the respective source packs by: - Translating the bit position (pseudo-pack order) to a MIDX position (lexical order). - Use the MIDX position to obtain the offset at which the given object occurs in the source pack. - Then translate that offset back into a pack relative position within the source pack by calling offset_to_pack_pos(). After doing this, then we can safely use the result as a pack position. Note that when doing single-pack reuse, as well as reusing objects from the MIDX's preferred pack, such translation is not necessary, since either ties are broken in favor of the preferred pack, or there are no ties to break at all (in the case of non-MIDX bitmaps). Failing to do this can result in strange failure modes. One example that can occur when misinterpreting bits in the above fashion is that Git thinks it's supposed to send a delta that the caller does not want. Under this (incorrect) assumption, we try to look up the delta's base (so that we can patch any OFS_DELTAs if necessary). We do this using find_reused_offset(). But if we try and call that function for an offset belonging to an object we did not send, we'll get back garbage. This can result in us computing a negative fixup value, which results in memory corruption when trying to write the (patched) OFS_DELTA header. Signed-off-by: Taylor Blau <me@ttaylorr.com> --- builtin/pack-objects.c | 44 ++++++++++++++++++++++++++++++------- t/t5332-multi-pack-reuse.sh | 31 ++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 12 deletions(-)