@@ -11,6 +11,7 @@ license that can be found in the LICENSE file or at
#include "constants.h"
#include "iter.h"
#include "pq.h"
+#include "reader.h"
#include "record.h"
#include "generic.h"
#include "reftable-merged.h"
@@ -25,7 +26,7 @@ struct merged_subiter {
struct merged_iter {
struct merged_subiter *subiters;
struct merged_iter_pqueue pq;
- size_t stack_len;
+ size_t subiters_len;
int suppress_deletions;
ssize_t advance_index;
};
@@ -38,12 +39,12 @@ static void merged_iter_init(struct merged_iter *mi,
mi->advance_index = -1;
mi->suppress_deletions = mt->suppress_deletions;
- REFTABLE_CALLOC_ARRAY(mi->subiters, mt->stack_len);
- for (size_t i = 0; i < mt->stack_len; i++) {
+ REFTABLE_CALLOC_ARRAY(mi->subiters, mt->readers_len);
+ for (size_t i = 0; i < mt->readers_len; i++) {
reftable_record_init(&mi->subiters[i].rec, typ);
- table_init_iter(&mt->stack[i], &mi->subiters[i].iter, typ);
+ reader_init_iter(mt->readers[i], &mi->subiters[i].iter, typ);
}
- mi->stack_len = mt->stack_len;
+ mi->subiters_len = mt->readers_len;
}
static void merged_iter_close(void *p)
@@ -51,7 +52,7 @@ static void merged_iter_close(void *p)
struct merged_iter *mi = p;
merged_iter_pqueue_release(&mi->pq);
- for (size_t i = 0; i < mi->stack_len; i++) {
+ for (size_t i = 0; i < mi->subiters_len; i++) {
reftable_iterator_destroy(&mi->subiters[i].iter);
reftable_record_release(&mi->subiters[i].rec);
}
@@ -80,7 +81,7 @@ static int merged_iter_seek(struct merged_iter *mi, struct reftable_record *want
mi->advance_index = -1;
- for (size_t i = 0; i < mi->stack_len; i++) {
+ for (size_t i = 0; i < mi->subiters_len; i++) {
err = iterator_seek(&mi->subiters[i].iter, want);
if (err < 0)
return err;
@@ -193,7 +194,7 @@ static void iterator_from_merged_iter(struct reftable_iterator *it,
}
int reftable_merged_table_new(struct reftable_merged_table **dest,
- struct reftable_table *stack, size_t n,
+ struct reftable_reader **readers, size_t n,
uint32_t hash_id)
{
struct reftable_merged_table *m = NULL;
@@ -201,10 +202,10 @@ int reftable_merged_table_new(struct reftable_merged_table **dest,
uint64_t first_min = 0;
for (size_t i = 0; i < n; i++) {
- uint64_t min = reftable_table_min_update_index(&stack[i]);
- uint64_t max = reftable_table_max_update_index(&stack[i]);
+ uint64_t min = reftable_reader_min_update_index(readers[i]);
+ uint64_t max = reftable_reader_max_update_index(readers[i]);
- if (reftable_table_hash_id(&stack[i]) != hash_id) {
+ if (reftable_reader_hash_id(readers[i]) != hash_id) {
return REFTABLE_FORMAT_ERROR;
}
if (i == 0 || min < first_min) {
@@ -216,8 +217,8 @@ int reftable_merged_table_new(struct reftable_merged_table **dest,
}
REFTABLE_CALLOC_ARRAY(m, 1);
- m->stack = stack;
- m->stack_len = n;
+ m->readers = readers;
+ m->readers_len = n;
m->min = first_min;
m->max = last_max;
m->hash_id = hash_id;
@@ -229,7 +230,6 @@ void reftable_merged_table_free(struct reftable_merged_table *mt)
{
if (!mt)
return;
- FREE_AND_NULL(mt->stack);
reftable_free(mt);
}
@@ -12,8 +12,8 @@ license that can be found in the LICENSE file or at
#include "system.h"
struct reftable_merged_table {
- struct reftable_table *stack;
- size_t stack_len;
+ struct reftable_reader **readers;
+ size_t readers_len;
uint32_t hash_id;
/* If unset, produce deletions. This is useful for compaction. For the
@@ -605,9 +605,9 @@ static void iterator_from_table_iter(struct reftable_iterator *it,
it->ops = &table_iter_vtable;
}
-static void reader_init_iter(struct reftable_reader *r,
- struct reftable_iterator *it,
- uint8_t typ)
+void reader_init_iter(struct reftable_reader *r,
+ struct reftable_iterator *it,
+ uint8_t typ)
{
struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
@@ -57,6 +57,10 @@ int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
void reader_close(struct reftable_reader *r);
const char *reader_name(struct reftable_reader *r);
+void reader_init_iter(struct reftable_reader *r,
+ struct reftable_iterator *it,
+ uint8_t typ);
+
/* initialize a block reader to read from `r` */
int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
uint64_t next_off, uint8_t want_typ);
@@ -28,13 +28,14 @@ struct reftable_merged_table;
/* A generic reftable; see below. */
struct reftable_table;
+struct reftable_reader;
/*
- * reftable_merged_table_new creates a new merged table. It takes ownership of
- * the stack array.
+ * reftable_merged_table_new creates a new merged table. The readers must be
+ * kept alive as long as the merged table is still in use.
*/
int reftable_merged_table_new(struct reftable_merged_table **dest,
- struct reftable_table *stack, size_t n,
+ struct reftable_reader **readers, size_t n,
uint32_t hash_id);
/* Initialize a merged table iterator for reading refs. */
@@ -225,13 +225,11 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
const char **names,
int reuse_open)
{
- size_t cur_len = !st->merged ? 0 : st->merged->stack_len;
+ size_t cur_len = !st->merged ? 0 : st->merged->readers_len;
struct reftable_reader **cur = stack_copy_readers(st, cur_len);
size_t names_len = names_length(names);
struct reftable_reader **new_readers =
reftable_calloc(names_len, sizeof(*new_readers));
- struct reftable_table *new_tables =
- reftable_calloc(names_len, sizeof(*new_tables));
size_t new_readers_len = 0;
struct reftable_merged_table *new_merged = NULL;
struct strbuf table_path = STRBUF_INIT;
@@ -267,17 +265,15 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
}
new_readers[new_readers_len] = rd;
- reftable_table_from_reader(&new_tables[new_readers_len], rd);
new_readers_len++;
}
/* success! */
- err = reftable_merged_table_new(&new_merged, new_tables,
+ err = reftable_merged_table_new(&new_merged, new_readers,
new_readers_len, st->opts.hash_id);
if (err < 0)
goto done;
- new_tables = NULL;
st->readers_len = new_readers_len;
if (st->merged)
reftable_merged_table_free(st->merged);
@@ -309,7 +305,6 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
reftable_reader_free(new_readers[i]);
}
reftable_free(new_readers);
- reftable_free(new_tables);
reftable_free(cur);
strbuf_release(&table_path);
return err;
@@ -520,7 +515,7 @@ static int stack_uptodate(struct reftable_stack *st)
}
}
- if (names[st->merged->stack_len]) {
+ if (names[st->merged->readers_len]) {
err = 1;
goto done;
}
@@ -659,7 +654,7 @@ int reftable_addition_commit(struct reftable_addition *add)
if (add->new_tables_len == 0)
goto done;
- for (i = 0; i < add->stack->merged->stack_len; i++) {
+ for (i = 0; i < add->stack->merged->readers_len; i++) {
strbuf_addstr(&table_list, add->stack->readers[i]->name);
strbuf_addstr(&table_list, "\n");
}
@@ -839,7 +834,7 @@ int reftable_addition_add(struct reftable_addition *add,
uint64_t reftable_stack_next_update_index(struct reftable_stack *st)
{
- int sz = st->merged->stack_len;
+ int sz = st->merged->readers_len;
if (sz > 0)
return reftable_reader_max_update_index(st->readers[sz - 1]) +
1;
@@ -906,30 +901,23 @@ static int stack_write_compact(struct reftable_stack *st,
size_t first, size_t last,
struct reftable_log_expiry_config *config)
{
- size_t subtabs_len = last - first + 1;
- struct reftable_table *subtabs = reftable_calloc(
- last - first + 1, sizeof(*subtabs));
struct reftable_merged_table *mt = NULL;
struct reftable_iterator it = { NULL };
struct reftable_ref_record ref = { NULL };
struct reftable_log_record log = { NULL };
+ size_t subtabs_len = last - first + 1;
uint64_t entries = 0;
int err = 0;
- for (size_t i = first, j = 0; i <= last; i++) {
- struct reftable_reader *t = st->readers[i];
- reftable_table_from_reader(&subtabs[j++], t);
- st->stats.bytes += t->size;
- }
+ for (size_t i = first; i <= last; i++)
+ st->stats.bytes += st->readers[i]->size;
reftable_writer_set_limits(wr, st->readers[first]->min_update_index,
st->readers[last]->max_update_index);
- err = reftable_merged_table_new(&mt, subtabs, subtabs_len,
+ err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len,
st->opts.hash_id);
- if (err < 0) {
- reftable_free(subtabs);
+ if (err < 0)
goto done;
- }
merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
err = reftable_iterator_seek_ref(&it, "");
@@ -1207,7 +1195,7 @@ static int stack_compact_range(struct reftable_stack *st,
* have compacted them.
*/
for (size_t j = 1; j < last - first + 1; j++) {
- const char *old = first + j < st->merged->stack_len ?
+ const char *old = first + j < st->merged->readers_len ?
st->readers[first + j]->name : NULL;
const char *new = names[i + j];
@@ -1248,10 +1236,10 @@ static int stack_compact_range(struct reftable_stack *st,
* `fd_read_lines()` uses a `NULL` sentinel to indicate that
* the array is at its end. As we use `free_names()` to free
* the array, we need to include this sentinel value here and
- * thus have to allocate `stack_len + 1` many entries.
+ * thus have to allocate `readers_len + 1` many entries.
*/
- REFTABLE_CALLOC_ARRAY(names, st->merged->stack_len + 1);
- for (size_t i = 0; i < st->merged->stack_len; i++)
+ REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1);
+ for (size_t i = 0; i < st->merged->readers_len; i++)
names[i] = xstrdup(st->readers[i]->name);
first_to_replace = first;
last_to_replace = last;
@@ -1358,7 +1346,7 @@ static int stack_compact_range_stats(struct reftable_stack *st,
int reftable_stack_compact_all(struct reftable_stack *st,
struct reftable_log_expiry_config *config)
{
- size_t last = st->merged->stack_len ? st->merged->stack_len - 1 : 0;
+ size_t last = st->merged->readers_len ? st->merged->readers_len - 1 : 0;
return stack_compact_range_stats(st, 0, last, config, 0);
}
@@ -1449,9 +1437,9 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st)
int overhead = header_size(version) - 1;
uint64_t *sizes;
- REFTABLE_CALLOC_ARRAY(sizes, st->merged->stack_len);
+ REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len);
- for (size_t i = 0; i < st->merged->stack_len; i++)
+ for (size_t i = 0; i < st->merged->readers_len; i++)
sizes[i] = st->readers[i]->size - overhead;
return sizes;
@@ -1461,7 +1449,7 @@ int reftable_stack_auto_compact(struct reftable_stack *st)
{
uint64_t *sizes = stack_table_sizes_for_compaction(st);
struct segment seg =
- suggest_compaction_segment(sizes, st->merged->stack_len,
+ suggest_compaction_segment(sizes, st->merged->readers_len,
st->opts.auto_compaction_factor);
reftable_free(sizes);
if (segment_size(&seg) > 0)
@@ -347,9 +347,9 @@ static void test_reftable_stack_transaction_api_performs_auto_compaction(void)
* all tables in the stack.
*/
if (i != n)
- EXPECT(st->merged->stack_len == i + 1);
+ EXPECT(st->merged->readers_len == i + 1);
else
- EXPECT(st->merged->stack_len == 1);
+ EXPECT(st->merged->readers_len == 1);
}
reftable_stack_destroy(st);
@@ -375,7 +375,7 @@ static void test_reftable_stack_auto_compaction_fails_gracefully(void)
err = reftable_stack_add(st, write_test_ref, &ref);
EXPECT_ERR(err);
- EXPECT(st->merged->stack_len == 1);
+ EXPECT(st->merged->readers_len == 1);
EXPECT(st->stats.attempts == 0);
EXPECT(st->stats.failures == 0);
@@ -390,7 +390,7 @@ static void test_reftable_stack_auto_compaction_fails_gracefully(void)
ref.update_index = 2;
err = reftable_stack_add(st, write_test_ref, &ref);
EXPECT_ERR(err);
- EXPECT(st->merged->stack_len == 2);
+ EXPECT(st->merged->readers_len == 2);
EXPECT(st->stats.attempts == 1);
EXPECT(st->stats.failures == 1);
@@ -881,7 +881,7 @@ static void test_reftable_stack_auto_compaction(void)
err = reftable_stack_auto_compact(st);
EXPECT_ERR(err);
- EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i));
+ EXPECT(i < 3 || st->merged->readers_len < 2 * fastlog2(i));
}
EXPECT(reftable_stack_compaction_stats(st)->entries_written <
@@ -905,7 +905,7 @@ static void test_reftable_stack_auto_compaction_with_locked_tables(void)
EXPECT_ERR(err);
write_n_ref_tables(st, 5);
- EXPECT(st->merged->stack_len == 5);
+ EXPECT(st->merged->readers_len == 5);
/*
* Given that all tables we have written should be roughly the same
@@ -925,7 +925,7 @@ static void test_reftable_stack_auto_compaction_with_locked_tables(void)
err = reftable_stack_auto_compact(st);
EXPECT_ERR(err);
EXPECT(st->stats.failures == 0);
- EXPECT(st->merged->stack_len == 4);
+ EXPECT(st->merged->readers_len == 4);
reftable_stack_destroy(st);
strbuf_release(&buf);
@@ -970,9 +970,9 @@ static void test_reftable_stack_add_performs_auto_compaction(void)
* all tables in the stack.
*/
if (i != n)
- EXPECT(st->merged->stack_len == i + 1);
+ EXPECT(st->merged->readers_len == i + 1);
else
- EXPECT(st->merged->stack_len == 1);
+ EXPECT(st->merged->readers_len == 1);
}
reftable_stack_destroy(st);
@@ -994,7 +994,7 @@ static void test_reftable_stack_compaction_with_locked_tables(void)
EXPECT_ERR(err);
write_n_ref_tables(st, 3);
- EXPECT(st->merged->stack_len == 3);
+ EXPECT(st->merged->readers_len == 3);
/* Lock one of the tables that we're about to compact. */
strbuf_reset(&buf);
@@ -1008,7 +1008,7 @@ static void test_reftable_stack_compaction_with_locked_tables(void)
err = reftable_stack_compact_all(st, NULL);
EXPECT(err == REFTABLE_LOCK_ERROR);
EXPECT(st->stats.failures == 1);
- EXPECT(st->merged->stack_len == 3);
+ EXPECT(st->merged->readers_len == 3);
reftable_stack_destroy(st);
strbuf_release(&buf);
@@ -94,10 +94,8 @@ merged_table_from_records(struct reftable_ref_record **refs,
struct strbuf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
- struct reftable_table *tabs;
int err;
- REFTABLE_CALLOC_ARRAY(tabs, n);
REFTABLE_CALLOC_ARRAY(*readers, n);
REFTABLE_CALLOC_ARRAY(*source, n);
@@ -108,10 +106,9 @@ merged_table_from_records(struct reftable_ref_record **refs,
err = reftable_new_reader(&(*readers)[i], &(*source)[i],
"name");
check(!err);
- reftable_table_from_reader(&tabs[i], (*readers)[i]);
}
- err = reftable_merged_table_new(&mt, tabs, n, GIT_SHA1_FORMAT_ID);
+ err = reftable_merged_table_new(&mt, *readers, n, GIT_SHA1_FORMAT_ID);
check(!err);
return mt;
}
@@ -272,10 +269,8 @@ merged_table_from_log_records(struct reftable_log_record **logs,
struct strbuf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
- struct reftable_table *tabs;
int err;
- REFTABLE_CALLOC_ARRAY(tabs, n);
REFTABLE_CALLOC_ARRAY(*readers, n);
REFTABLE_CALLOC_ARRAY(*source, n);
@@ -286,10 +281,9 @@ merged_table_from_log_records(struct reftable_log_record **logs,
err = reftable_new_reader(&(*readers)[i], &(*source)[i],
"name");
check(!err);
- reftable_table_from_reader(&tabs[i], (*readers)[i]);
}
- err = reftable_merged_table_new(&mt, tabs, n, GIT_SHA1_FORMAT_ID);
+ err = reftable_merged_table_new(&mt, *readers, n, GIT_SHA1_FORMAT_ID);
check(!err);
return mt;
}
@@ -418,7 +412,6 @@ static void t_default_write_opts(void)
};
int err;
struct reftable_block_source source = { 0 };
- struct reftable_table *tab = reftable_calloc(1, sizeof(*tab));
uint32_t hash_id;
struct reftable_reader *rd = NULL;
struct reftable_merged_table *merged = NULL;
@@ -440,10 +433,9 @@ static void t_default_write_opts(void)
hash_id = reftable_reader_hash_id(rd);
check_int(hash_id, ==, GIT_SHA1_FORMAT_ID);
- reftable_table_from_reader(&tab[0], rd);
- err = reftable_merged_table_new(&merged, tab, 1, GIT_SHA256_FORMAT_ID);
+ err = reftable_merged_table_new(&merged, &rd, 1, GIT_SHA256_FORMAT_ID);
check_int(err, ==, REFTABLE_FORMAT_ERROR);
- err = reftable_merged_table_new(&merged, tab, 1, GIT_SHA1_FORMAT_ID);
+ err = reftable_merged_table_new(&merged, &rd, 1, GIT_SHA1_FORMAT_ID);
check(!err);
reftable_reader_free(rd);
The merged table provides access to a reftable stack by merging the contents of those tables into a virtual table. These subtables are being tracked via `struct reftable_table`, which is a generic interface for accessing either a single reftable or a merged reftable. So in theory, it would be possible for the merged table to merge together other merged tables. This is somewhat nonsensical though: we only ever set up a merged table over normal reftables, and there is no reason to do otherwise. This generic interface thus makes the code way harder to follow and reason about than really necessary. The abstraction layer may also have an impact on performance, even though the extra set of vtable function calls probably doesn't really matter. Refactor the merged tables to use a `struct reftable_reader` for each of the subtables instead, which gives us direct access to the underlying tables. Adjust names accordingly. Signed-off-by: Patrick Steinhardt <ps@pks.im> --- reftable/merged.c | 28 +++++++++---------- reftable/merged.h | 4 +-- reftable/reader.c | 6 ++-- reftable/reader.h | 4 +++ reftable/reftable-merged.h | 7 +++-- reftable/stack.c | 48 ++++++++++++-------------------- reftable/stack_test.c | 22 +++++++-------- t/unit-tests/t-reftable-merged.c | 16 +++-------- 8 files changed, 60 insertions(+), 75 deletions(-)