| Message ID | 30e2e33479a6b6b07a4652e84eebd78f71fb363a.1714630191.git.ps@pks.im (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | reftable: expose write options as config | expand |
Patrick Steinhardt <ps@pks.im> writes: > When auto-compacting, the reftable library packs references such that > the sizes of the tables form a geometric sequence. The factor for this > geometric sequence is hardcoded to 2 right now. We're about to expose > this as a config option though, so let's expose the factor via write > options. > > Signed-off-by: Patrick Steinhardt <ps@pks.im> > --- > reftable/reftable-writer.h | 6 ++++++ > reftable/stack.c | 13 +++++++++---- > reftable/stack.h | 3 ++- > reftable/stack_test.c | 4 ++-- > 4 files changed, 19 insertions(+), 7 deletions(-) > > diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h > index 4cd8ebe6c7..155457b042 100644 > --- a/reftable/reftable-writer.h > +++ b/reftable/reftable-writer.h > @@ -49,6 +49,12 @@ struct reftable_write_options { > > /* boolean: Prevent auto-compaction of tables. */ > unsigned disable_auto_compact : 1; > + > + /* > + * Geometric sequence factor used by auto-compaction to decide which > + * tables to compact. Defaults to 2 if unset. > + */ > + uint8_t auto_compaction_factor; > }; > > /* reftable_block_stats holds statistics for a single block type */ > diff --git a/reftable/stack.c b/reftable/stack.c > index 7b4fff7c9e..6b0f8e13e7 100644 > --- a/reftable/stack.c > +++ b/reftable/stack.c > @@ -1215,12 +1215,16 @@ static int segment_size(struct segment *s) > return s->end - s->start; > } > > -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) > +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, > + uint8_t factor) > { > struct segment seg = { 0 }; > uint64_t bytes; > size_t i; > > + if (!factor) > + factor = 2; > + This should probably go in reftable/constants.h [snip]
On Fri, May 10, 2024 at 04:55:10AM -0500, Karthik Nayak wrote: > Patrick Steinhardt <ps@pks.im> writes: [snip] > > diff --git a/reftable/stack.c b/reftable/stack.c > > index 7b4fff7c9e..6b0f8e13e7 100644 > > --- a/reftable/stack.c > > +++ b/reftable/stack.c > > @@ -1215,12 +1215,16 @@ static int segment_size(struct segment *s) > > return s->end - s->start; > > } > > > > -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) > > +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, > > + uint8_t factor) > > { > > struct segment seg = { 0 }; > > uint64_t bytes; > > size_t i; > > > > + if (!factor) > > + factor = 2; > > + > > This should probably go in reftable/constants.h Good idea, will do. Patrick
diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index 4cd8ebe6c7..155457b042 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -49,6 +49,12 @@ struct reftable_write_options { /* boolean: Prevent auto-compaction of tables. */ unsigned disable_auto_compact : 1; + + /* + * Geometric sequence factor used by auto-compaction to decide which + * tables to compact. Defaults to 2 if unset. + */ + uint8_t auto_compaction_factor; }; /* reftable_block_stats holds statistics for a single block type */ diff --git a/reftable/stack.c b/reftable/stack.c index 7b4fff7c9e..6b0f8e13e7 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -1215,12 +1215,16 @@ static int segment_size(struct segment *s) return s->end - s->start; } -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, + uint8_t factor) { struct segment seg = { 0 }; uint64_t bytes; size_t i; + if (!factor) + factor = 2; + /* * If there are no tables or only a single one then we don't have to * compact anything. The sequence is geometric by definition already. @@ -1252,7 +1256,7 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) * 64, 32, 16, 8, 4, 3, 1 */ for (i = n - 1; i > 0; i--) { - if (sizes[i - 1] < sizes[i] * 2) { + if (sizes[i - 1] < sizes[i] * factor) { seg.end = i + 1; bytes = sizes[i]; break; @@ -1278,7 +1282,7 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) uint64_t curr = bytes; bytes += sizes[i - 1]; - if (sizes[i - 1] < curr * 2) { + if (sizes[i - 1] < curr * factor) { seg.start = i - 1; seg.bytes = bytes; } @@ -1304,7 +1308,8 @@ int reftable_stack_auto_compact(struct reftable_stack *st) { uint64_t *sizes = stack_table_sizes_for_compaction(st); struct segment seg = - suggest_compaction_segment(sizes, st->merged->stack_len); + suggest_compaction_segment(sizes, st->merged->stack_len, + st->opts.auto_compaction_factor); reftable_free(sizes); if (segment_size(&seg) > 0) return stack_compact_range_stats(st, seg.start, seg.end - 1, diff --git a/reftable/stack.h b/reftable/stack.h index 97d7ebc043..5b45cff4f7 100644 --- a/reftable/stack.h +++ b/reftable/stack.h @@ -35,6 +35,7 @@ struct segment { uint64_t bytes; }; -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n); +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, + uint8_t factor); #endif diff --git a/reftable/stack_test.c b/reftable/stack_test.c index 3316d55f19..f6c11ef18d 100644 --- a/reftable/stack_test.c +++ b/reftable/stack_test.c @@ -767,7 +767,7 @@ static void test_suggest_compaction_segment(void) { uint64_t sizes[] = { 512, 64, 17, 16, 9, 9, 9, 16, 2, 16 }; struct segment min = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); EXPECT(min.start == 1); EXPECT(min.end == 10); } @@ -776,7 +776,7 @@ static void test_suggest_compaction_segment_nothing(void) { uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; struct segment result = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); EXPECT(result.start == result.end); }
When auto-compacting, the reftable library packs references such that the sizes of the tables form a geometric sequence. The factor for this geometric sequence is hardcoded to 2 right now. We're about to expose this as a config option though, so let's expose the factor via write options. Signed-off-by: Patrick Steinhardt <ps@pks.im> --- reftable/reftable-writer.h | 6 ++++++ reftable/stack.c | 13 +++++++++---- reftable/stack.h | 3 ++- reftable/stack_test.c | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-)