@@ -66,6 +66,8 @@ static int merged_iter_seek(struct merged_iter *mi, struct reftable_record *want
int err;
mi->advance_index = -1;
+ while (!merged_iter_pqueue_is_empty(mi->pq))
+ merged_iter_pqueue_remove(&mi->pq);
for (size_t i = 0; i < mi->subiters_len; i++) {
err = iterator_seek(&mi->subiters[i].iter, want);
@@ -273,6 +273,78 @@ static void t_merged_seek_multiple_times(void)
reftable_free(sources);
}
+static void t_merged_seek_multiple_times_without_draining(void)
+{
+ struct reftable_ref_record r1[] = {
+ {
+ .refname = (char *) "a",
+ .update_index = 1,
+ .value_type = REFTABLE_REF_VAL1,
+ .value.val1 = { 1 },
+ },
+ {
+ .refname = (char *) "c",
+ .update_index = 1,
+ .value_type = REFTABLE_REF_VAL1,
+ .value.val1 = { 2 },
+ }
+ };
+ struct reftable_ref_record r2[] = {
+ {
+ .refname = (char *) "b",
+ .update_index = 2,
+ .value_type = REFTABLE_REF_VAL1,
+ .value.val1 = { 3 },
+ },
+ {
+ .refname = (char *) "d",
+ .update_index = 2,
+ .value_type = REFTABLE_REF_VAL1,
+ .value.val1 = { 4 },
+ },
+ };
+ struct reftable_ref_record *refs[] = {
+ r1, r2,
+ };
+ size_t sizes[] = {
+ ARRAY_SIZE(r1), ARRAY_SIZE(r2),
+ };
+ struct reftable_buf bufs[] = {
+ REFTABLE_BUF_INIT, REFTABLE_BUF_INIT,
+ };
+ struct reftable_block_source *sources = NULL;
+ struct reftable_reader **readers = NULL;
+ struct reftable_ref_record rec = { 0 };
+ struct reftable_iterator it = { 0 };
+ struct reftable_merged_table *mt;
+ int err;
+
+ mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2);
+ merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
+
+ err = reftable_iterator_seek_ref(&it, "b");
+ check(!err);
+ err = reftable_iterator_next_ref(&it, &rec);
+ check(!err);
+ err = reftable_ref_record_equal(&rec, &r2[0], REFTABLE_HASH_SIZE_SHA1);
+ check(err == 1);
+
+ err = reftable_iterator_seek_ref(&it, "a");
+ check(!err);
+ err = reftable_iterator_next_ref(&it, &rec);
+ check(!err);
+ err = reftable_ref_record_equal(&rec, &r1[0], REFTABLE_HASH_SIZE_SHA1);
+ check(err == 1);
+
+ for (size_t i = 0; i < ARRAY_SIZE(bufs); i++)
+ reftable_buf_release(&bufs[i]);
+ readers_destroy(readers, ARRAY_SIZE(refs));
+ reftable_ref_record_release(&rec);
+ reftable_iterator_destroy(&it);
+ reftable_merged_table_free(mt);
+ reftable_free(sources);
+}
+
static struct reftable_merged_table *
merged_table_from_log_records(struct reftable_log_record **logs,
struct reftable_block_source **source,
@@ -467,6 +539,7 @@ int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
TEST(t_merged_logs(), "merged table with multiple log updates for same ref");
TEST(t_merged_refs(), "merged table with multiple updates to same ref");
TEST(t_merged_seek_multiple_times(), "merged table can seek multiple times");
+ TEST(t_merged_seek_multiple_times_without_draining(), "merged table can seek multiple times without draining");
TEST(t_merged_single_record(), "ref occurring in only one record can be fetched");
return test_done();
In 5bf96e0c39 (reftable/generic: move seeking of records into the iterator, 2024-05-13) we have refactored the reftable codebase such that iterators can be initialized once and then re-seeked multiple times. This feature is used by 1869525066 (refs/reftable: wire up support for exclude patterns, 2024-09-16) in order to skip records based on exclude patterns provided by the caller. The logic to re-seek the merged iterator is insufficient though because we don't drain the priority queue on a re-seek. This means that the queue may contain stale entries and thus reading the next record in the queue will return the wrong entry. While this is an obvious bug, it is harmless in the context of above exclude patterns: - If the queue contained stale entries that match the pattern then the caller would already know to filter out such refs. This is because our codebase is prepared to handle backends that don't have a way to efficiently implement exclude patterns. - If the queue contained stale entries that don't match the pattern we'd eventually filter out any duplicates. This is because the reftable code discards items with the same ref name and sorts any remaining entries properly. So things happen to work in this context regardless of the bug, and there is no other use case yet where we re-seek iterators. We're about to introduce a caching mechanism though where iterators are reused by the reftable backend, and that will expose the bug. Fix the issue by draining the priority queue when seeking and add a testcase that surfaces the issue. Signed-off-by: Patrick Steinhardt <ps@pks.im> --- reftable/merged.c | 2 ++ t/unit-tests/t-reftable-merged.c | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+)