You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/10/12 03:41:59 UTC
(skywalking-banyandb) branch main updated: feat(index): narrow a native time range before projecting a segment (#1405)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new fade662e9 feat(index): narrow a native time range before projecting a segment (#1405)
fade662e9 is described below
commit fade662e9bffebfd85bc3caecb750fac3ea2ae12
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Oct 12 11:41:54 2026 +0800
feat(index): narrow a native time range before projecting a segment (#1405)
* fix(index): give merged native segments exact time bounds
The streaming merge wrote 0 for both footer time slots, so every merged
segment claimed to have no timestamps, and so did its manifest entry once
persisted. Merged segments soon hold most of an index's data, which makes
them the segments a query most wants to skip or narrow by time, so this
blocked the whole time-pruning direction: a query could not exclude a merged
segment, nor drop the per-document time check for one it fully covers.
---
CHANGES.md | 1 -
banyand/internal/storage/index.go | 1 +
banyand/liaison/grpc/measure.go | 6 -
banyand/liaison/grpc/measure_write_test.go | 32 --
banyand/liaison/grpc/schema_revision.go | 35 --
banyand/liaison/grpc/stream.go | 6 -
banyand/liaison/grpc/stream_write_test.go | 32 --
banyand/liaison/grpc/trace.go | 6 -
banyand/liaison/grpc/trace_write_test.go | 34 --
banyand/liaison/grpc/write_helpers_test.go | 9 -
banyand/stream/index.go | 5 +-
banyand/stream/native_index_integration_test.go | 8 +-
banyand/stream/tstable.go | 3 +-
banyand/stream/tstable_test.go | 4 +-
docs/design/0.12.0/README.md | 1 +
.../0.12.0/native-index-time-pruning/README.md | 465 +++++++++++++++++
pkg/index/internal/nativeice/encode.go | 8 +
pkg/index/internal/nativeice/merge.go | 20 +-
pkg/index/internal/nativeice/merge_sections.go | 26 +
pkg/index/internal/nativeice/merge_stream.go | 16 +-
.../internal/nativeice/merge_time_bounds_test.go | 317 ++++++++++++
pkg/index/internal/nativeice/nativeice.go | 12 +
pkg/index/internal/nativeice/numeric.go | 153 +++++-
pkg/index/internal/nativeice/numeric_cover_test.go | 204 ++++++++
pkg/index/internal/nativeice/numeric_range_test.go | 268 ++++++++++
pkg/index/metrics/metrics.go | 60 +++
pkg/index/native/merge_staging.go | 4 +
pkg/index/native/owner.go | 102 +++-
pkg/index/native/query.go | 116 ++++-
pkg/index/native/query_test.go | 43 ++
pkg/index/native/sort_cursor.go | 144 ++++--
pkg/index/native/time_prune.go | 425 ++++++++++++++++
pkg/index/native/time_prune_bench_test.go | 441 ++++++++++++++++
pkg/index/native/time_prune_parity_test.go | 562 +++++++++++++++++++++
pkg/index/native/time_prune_test.go | 271 ++++++++++
pkg/index/nativeadapter/searcher.go | 6 +
pkg/index/nativeadapter/store.go | 1 +
pkg/test/eventually.go | 30 ++
pkg/test/eventually_test.go | 54 ++
test/cases/measure/data/data.go | 207 ++++----
test/cases/property/data/data.go | 11 +-
test/cases/schema/apply_rollback.go | 28 +-
test/cases/schema/clamp.go | 76 ++-
test/cases/schema/deletion.go | 220 ++++++--
test/cases/schema/query_gate.go | 56 +-
test/cases/schema/shape_break.go | 102 ++--
test/cases/schema/tag_family_move.go | 145 ++++--
test/cases/schema/write_gate.go | 12 +-
test/cases/stream/data/data.go | 139 +++--
test/cases/trace/data/data.go | 127 +++--
test/cases/tracepipeline/data/data.go | 44 +-
test/e2e/tracepipeline/e2e_test.go | 65 ++-
test/integration/distributed/backup/common.go | 25 +-
test/integration/distributed/deletion/common.go | 89 +++-
test/integration/distributed/inspection/common.go | 292 ++++++++---
test/integration/distributed/rbac/rbac_test.go | 51 +-
test/integration/handoff/handoff_suite_test.go | 57 ++-
test/integration/standalone/backup/common.go | 29 +-
test/integration/standalone/deletion/common.go | 83 ++-
test/integration/standalone/inspection/common.go | 280 +++++++---
.../standalone/other/rbac_data/rbac_data_test.go | 123 ++++-
test/property_repair/shared_utils.go | 35 +-
62 files changed, 5324 insertions(+), 903 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 59e507e50..47c75031f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -26,7 +26,6 @@ Release Notes.
### Bug Fixes
-- Stop the liaison from refusing a Measure, Stream or Trace write with `STATUS_NOT_FOUND` right after a schema barrier reported the schema applied. The barrier's watermark advances by each processed event's own revision, so an event handled out of revision order could carry it past a schema whose handlers had not yet filled the liaison's entity cache. A write that carries a schema revision now waits for that cache, bounded by the larger of `--{measure,stream,trace}-metadata-cache-wait-du [...]
- Fix a data race in Measure TopN where a pooled TopN value reused for a query wrote its tag names into the streaming processor's entity tag name slice.
- Stop `pkg/index/native`'s compaction-persistence test from flaking on `Test Pkg`: it asserted that garbage collection succeeds the instant a read view closes, without accounting for a background persist still holding an older root.
- Stamp `RELEASE_VERSION` into official release binaries so `--version` reports the release instead of `-`.
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index 786a8ad4d..3e0738af9 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -138,6 +138,7 @@ func newSeriesIndex(ctx context.Context, root string, flushTimeoutSeconds int64,
PresenceCacheBytes: cacheMaxBytes,
IdentifierDocValues: true,
PersistInterval: persistInterval,
+ TimeMetrics: si.metrics.NativeTimeMetrics(si.p.SegLabelValues()...),
})
if err != nil {
return nil, err
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index b1637758a..8a46416cb 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -193,12 +193,6 @@ func (ms *measureService) validateWriteRequest(writeRequest *measurev1.WriteRequ
if metadata.ModRevision > 0 {
id := getID(metadata)
measureCache, existed := ms.entityRepo.getLocator(id)
- if !existed && awaitSchemaPresent(func() bool {
- _, found := ms.entityRepo.getLocator(id)
- return found
- }, schemaPresenceWait(ms.maxWaitDuration)) {
- measureCache, existed = ms.entityRepo.getLocator(id)
- }
if !existed {
ms.l.Error().Stringer("written", writeRequest).Msg("measure schema not found")
ms.sendReply(metadata, modelv1.Status_STATUS_NOT_FOUND, writeRequest.GetMessageId(), measure)
diff --git a/banyand/liaison/grpc/measure_write_test.go b/banyand/liaison/grpc/measure_write_test.go
index 24ac32de1..e866e89cf 100644
--- a/banyand/liaison/grpc/measure_write_test.go
+++ b/banyand/liaison/grpc/measure_write_test.go
@@ -130,35 +130,3 @@ func TestValidateWriteRequest_Measure_ZeroRevision_SkipsCheck(t *testing.T) {
assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
assert.Empty(t, mock.replies)
}
-
-// TestValidateWriteRequest_Measure_SchemaArrivesLate_ReturnsSucceed verifies that a
-// write naming a schema revision waits for the entity cache to learn that schema
-// instead of failing with STATUS_NOT_FOUND, even with no metadata-cache wait set.
-func TestValidateWriteRequest_Measure_SchemaArrivesLate_ReturnsSucceed(t *testing.T) {
- withSchemaPresenceGrace(t, 500*time.Millisecond)
- id := identity{group: "g", name: "m"}
- er := newEmptyEntityRepo()
- svc := newTestMeasureService(er, 0)
- mock := &mockBidiServer[measurev1.WriteRequest, measurev1.WriteResponse]{}
-
- advanceLocatorAfter(er, id, seededLocatorRepoRev, 30*time.Millisecond)
-
- meta := &commonv1.Metadata{Group: "g", Name: "m", ModRevision: seededLocatorRepoRev}
- st := svc.validateWriteRequest(validMeasureWriteRequest(), meta, mock)
-
- assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
- assert.Empty(t, mock.replies)
-}
-
-// TestValidateWriteRequest_Measure_SchemaNeverArrives_ReturnsNotFound verifies that
-// the presence wait is bounded: a schema the cache never learns is still refused.
-func TestValidateWriteRequest_Measure_SchemaNeverArrives_ReturnsNotFound(t *testing.T) {
- withSchemaPresenceGrace(t, 30*time.Millisecond)
- svc := newTestMeasureService(newEmptyEntityRepo(), 0)
- mock := &mockBidiServer[measurev1.WriteRequest, measurev1.WriteResponse]{}
-
- meta := &commonv1.Metadata{Group: "g", Name: "m", ModRevision: seededLocatorRepoRev}
- st := svc.validateWriteRequest(validMeasureWriteRequest(), meta, mock)
-
- assert.Equal(t, modelv1.Status_STATUS_NOT_FOUND, st)
-}
diff --git a/banyand/liaison/grpc/schema_revision.go b/banyand/liaison/grpc/schema_revision.go
index 1c484c895..ba902f6a6 100644
--- a/banyand/liaison/grpc/schema_revision.go
+++ b/banyand/liaison/grpc/schema_revision.go
@@ -111,41 +111,6 @@ func checkQueryGate(
return groupStatuses, false
}
-// schemaPresenceGrace is the minimum time a write that carries a schema
-// revision waits for this liaison's entity cache to learn that schema before
-// it is refused with STATUS_NOT_FOUND. It matches barrierDefaultTimeout.
-var schemaPresenceGrace = barrierDefaultTimeout
-
-// awaitSchemaPresent polls present until it reports true or maxWait elapses.
-//
-// A write with ModRevision > 0 names a schema the registry has acknowledged,
-// but the entity cache is filled by a schema-event handler that can still be
-// pending when the client's barrier returns: the barrier's watermark advances
-// by each processed event's own revision, so an event processed out of
-// revision order can carry it past a revision whose handlers have not run.
-// Waiting here closes that window; a schema that never arrives still ends in
-// STATUS_NOT_FOUND once maxWait elapses.
-func awaitSchemaPresent(present func() bool, maxWait time.Duration) bool {
- if present() {
- return true
- }
- return awaitRevisionReached(func() int64 {
- if present() {
- return 1
- }
- return 0
- }, 1, maxWait)
-}
-
-// schemaPresenceWait returns the bound for awaitSchemaPresent: the configured
-// metadata-cache wait, but never less than schemaPresenceGrace.
-func schemaPresenceWait(configured time.Duration) time.Duration {
- if configured > schemaPresenceGrace {
- return configured
- }
- return schemaPresenceGrace
-}
-
// awaitRevisionReached polls getRevision until the returned value is >= target or maxWait elapses.
// Returns true when the target revision was observed, false on timeout.
// The initial check is performed before any sleep, so a cache that is already at target never
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 0e0f921f6..a45f9533a 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -102,12 +102,6 @@ func (s *streamService) validateWriteRequest(writeEntity *streamv1.WriteRequest,
if metadata.ModRevision > 0 {
id := getID(metadata)
streamCache, existed := s.entityRepo.getLocator(id)
- if !existed && awaitSchemaPresent(func() bool {
- _, found := s.entityRepo.getLocator(id)
- return found
- }, schemaPresenceWait(s.maxWaitDuration)) {
- streamCache, existed = s.entityRepo.getLocator(id)
- }
if !existed {
s.l.Error().Stringer("written", writeEntity).Msg("stream schema not found")
s.sendReply(metadata, modelv1.Status_STATUS_NOT_FOUND, writeEntity.GetMessageId(), stream)
diff --git a/banyand/liaison/grpc/stream_write_test.go b/banyand/liaison/grpc/stream_write_test.go
index 11c159b96..31af53b73 100644
--- a/banyand/liaison/grpc/stream_write_test.go
+++ b/banyand/liaison/grpc/stream_write_test.go
@@ -130,35 +130,3 @@ func TestValidateWriteRequest_Stream_ZeroRevision_SkipsCheck(t *testing.T) {
assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
assert.Empty(t, mock.replies)
}
-
-// TestValidateWriteRequest_Stream_SchemaArrivesLate_ReturnsSucceed verifies that a
-// write naming a schema revision waits for the entity cache to learn that schema
-// instead of failing with STATUS_NOT_FOUND, even with no metadata-cache wait set.
-func TestValidateWriteRequest_Stream_SchemaArrivesLate_ReturnsSucceed(t *testing.T) {
- withSchemaPresenceGrace(t, 500*time.Millisecond)
- id := identity{group: "g", name: "s"}
- er := newEmptyEntityRepo()
- svc := newTestStreamService(er, 0)
- mock := &mockBidiServer[streamv1.WriteRequest, streamv1.WriteResponse]{}
-
- advanceLocatorAfter(er, id, seededLocatorRepoRev, 30*time.Millisecond)
-
- meta := &commonv1.Metadata{Group: "g", Name: "s", ModRevision: seededLocatorRepoRev}
- st := svc.validateWriteRequest(validStreamWriteRequest(), meta, mock)
-
- assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
- assert.Empty(t, mock.replies)
-}
-
-// TestValidateWriteRequest_Stream_SchemaNeverArrives_ReturnsNotFound verifies that
-// the presence wait is bounded: a schema the cache never learns is still refused.
-func TestValidateWriteRequest_Stream_SchemaNeverArrives_ReturnsNotFound(t *testing.T) {
- withSchemaPresenceGrace(t, 30*time.Millisecond)
- svc := newTestStreamService(newEmptyEntityRepo(), 0)
- mock := &mockBidiServer[streamv1.WriteRequest, streamv1.WriteResponse]{}
-
- meta := &commonv1.Metadata{Group: "g", Name: "s", ModRevision: seededLocatorRepoRev}
- st := svc.validateWriteRequest(validStreamWriteRequest(), meta, mock)
-
- assert.Equal(t, modelv1.Status_STATUS_NOT_FOUND, st)
-}
diff --git a/banyand/liaison/grpc/trace.go b/banyand/liaison/grpc/trace.go
index 3e80c88e9..ed0313504 100644
--- a/banyand/liaison/grpc/trace.go
+++ b/banyand/liaison/grpc/trace.go
@@ -107,12 +107,6 @@ func (s *traceService) validateWriteRequest(writeEntity *tracev1.WriteRequest,
) modelv1.Status {
id := getID(metadata)
traceEntity, existed := s.entityRepo.getTrace(id)
- if !existed && metadata.ModRevision > 0 && awaitSchemaPresent(func() bool {
- _, found := s.entityRepo.getTrace(id)
- return found
- }, schemaPresenceWait(s.maxWaitDuration)) {
- traceEntity, existed = s.entityRepo.getTrace(id)
- }
if !existed {
s.l.Error().Stringer("written", writeEntity).Msg("trace schema not found")
s.sendReply(metadata, modelv1.Status_STATUS_NOT_FOUND, writeEntity.GetVersion(), stream)
diff --git a/banyand/liaison/grpc/trace_write_test.go b/banyand/liaison/grpc/trace_write_test.go
index 08fb59e57..989ea298c 100644
--- a/banyand/liaison/grpc/trace_write_test.go
+++ b/banyand/liaison/grpc/trace_write_test.go
@@ -136,37 +136,3 @@ func TestValidateWriteRequest_Trace_ZeroRevision_SkipsCheck(t *testing.T) {
assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
assert.Empty(t, mock.replies)
}
-
-// TestValidateWriteRequest_Trace_SchemaArrivesLate_ReturnsSucceed verifies that a
-// write naming a schema revision waits for the entity cache to learn that schema
-// instead of failing with STATUS_NOT_FOUND, even with no metadata-cache wait set.
-func TestValidateWriteRequest_Trace_SchemaArrivesLate_ReturnsSucceed(t *testing.T) {
- withSchemaPresenceGrace(t, 500*time.Millisecond)
- id := identity{group: "g", name: "t"}
- er := newEmptyEntityRepo()
- svc := newTestTraceService(er, 0)
- mock := &mockBidiServer[tracev1.WriteRequest, tracev1.WriteResponse]{}
-
- advanceTraceRevAfter(er, id, 100, 20*time.Millisecond)
-
- meta := &commonv1.Metadata{Group: "g", Name: "t", ModRevision: 100}
- st := svc.validateWriteRequest(validTraceWriteRequest(), meta, nil, mock)
-
- assert.Equal(t, modelv1.Status_STATUS_SUCCEED, st)
- assert.Empty(t, mock.replies)
-}
-
-// TestValidateWriteRequest_Trace_SchemaNeverArrives_ReturnsNotFound verifies that
-// the presence wait is bounded, and that a write without a revision is refused at
-// once, as before.
-func TestValidateWriteRequest_Trace_SchemaNeverArrives_ReturnsNotFound(t *testing.T) {
- withSchemaPresenceGrace(t, 30*time.Millisecond)
- svc := newTestTraceService(newEmptyEntityRepo(), 0)
-
- for _, revision := range []int64{100, 0} {
- mock := &mockBidiServer[tracev1.WriteRequest, tracev1.WriteResponse]{}
- meta := &commonv1.Metadata{Group: "g", Name: "t", ModRevision: revision}
- st := svc.validateWriteRequest(validTraceWriteRequest(), meta, nil, mock)
- assert.Equal(t, modelv1.Status_STATUS_NOT_FOUND, st, "revision %d", revision)
- }
-}
diff --git a/banyand/liaison/grpc/write_helpers_test.go b/banyand/liaison/grpc/write_helpers_test.go
index 2ad0cd3eb..bc789a507 100644
--- a/banyand/liaison/grpc/write_helpers_test.go
+++ b/banyand/liaison/grpc/write_helpers_test.go
@@ -21,7 +21,6 @@ import (
"context"
"io"
"sync"
- "testing"
"time"
grpcmd "google.golang.org/grpc/metadata"
@@ -117,11 +116,3 @@ func advanceTraceRevAfter(er *entityRepo, id identity, newRev int64, delay time.
er.RWMutex.Unlock()
}()
}
-
-// withSchemaPresenceGrace overrides schemaPresenceGrace for one test.
-func withSchemaPresenceGrace(t *testing.T, grace time.Duration) {
- t.Helper()
- previous := schemaPresenceGrace
- schemaPresenceGrace = grace
- t.Cleanup(func() { schemaPresenceGrace = previous })
-}
diff --git a/banyand/stream/index.go b/banyand/stream/index.go
index fc6b51b4c..2e10e8813 100644
--- a/banyand/stream/index.go
+++ b/banyand/stream/index.go
@@ -43,7 +43,9 @@ type elementIndex struct {
location string
}
-func newElementIndex(ctx context.Context, root string, flushTimeoutSeconds int64, idxMetrics *idxmetrics.Metrics, leases ...storage.RootLease) (*elementIndex, error) {
+func newElementIndex(ctx context.Context, root string, flushTimeoutSeconds int64, idxMetrics *idxmetrics.Metrics,
+ labelValues []string, leases ...storage.RootLease,
+) (*elementIndex, error) {
if len(leases) != 1 || leases[0] == nil {
return nil, errors.New("element index: exactly one root lease is required")
}
@@ -67,6 +69,7 @@ func newElementIndex(ctx context.Context, root string, flushTimeoutSeconds int64
MaxTerms: nativeadapter.DefaultMaxTerms,
AsyncPersistence: flushTimeoutSeconds > 0,
PersistInterval: time.Duration(flushTimeoutSeconds) * time.Second,
+ TimeMetrics: idxMetrics.NativeTimeMetrics(labelValues...),
}); err != nil {
return nil, err
}
diff --git a/banyand/stream/native_index_integration_test.go b/banyand/stream/native_index_integration_test.go
index 09b9ce10c..113189d00 100644
--- a/banyand/stream/native_index_integration_test.go
+++ b/banyand/stream/native_index_integration_test.go
@@ -31,12 +31,12 @@ import (
func TestNativeElementIndexRequiresLeaseAndWrites(t *testing.T) {
root := t.TempDir()
- _, err := newElementIndex(context.Background(), root, 0, nil)
+ _, err := newElementIndex(context.Background(), root, 0, nil, nil)
require.Error(t, err)
_, statErr := os.Stat(filepath.Join(root, elementIndexFilename))
require.ErrorIs(t, statErr, os.ErrNotExist)
lease := newTestRootLease(t, root)
- element, err := newElementIndex(context.Background(), root, 0, nil, lease)
+ element, err := newElementIndex(context.Background(), root, 0, nil, nil, lease)
require.NoError(t, err)
field := index.NewStringField(index.FieldKey{IndexRuleID: 1, SeriesID: 1}, "ok")
field.Store = true
@@ -58,7 +58,7 @@ func TestNativeElementIndexRequiresLeaseAndWrites(t *testing.T) {
require.Equal(t, map[string][][]byte{"\x00\x00\x00\x01": {[]byte("ok")}}, stored)
require.NoError(t, snapshotReader.Close())
require.NoError(t, element.Close())
- element, err = newElementIndex(context.Background(), root, 0, nil, lease)
+ element, err = newElementIndex(context.Background(), root, 0, nil, nil, lease)
require.NoError(t, err)
defer element.Close()
iterator, err = element.Sort(context.Background(), []common.SeriesID{1}, field.Key, modelv1.Sort_SORT_ASC, nil, 1)
@@ -70,7 +70,7 @@ func TestNativeElementIndexRequiresLeaseAndWrites(t *testing.T) {
func TestNativeElementIndexAdmitsStoredRowsAfterRequestCancellation(t *testing.T) {
root := t.TempDir()
- element, err := newElementIndex(context.Background(), root, 0, nil, newTestRootLease(t, root))
+ element, err := newElementIndex(context.Background(), root, 0, nil, nil, newTestRootLease(t, root))
require.NoError(t, err)
defer element.Close()
field := index.NewStringField(index.FieldKey{IndexRuleID: 1, SeriesID: 1}, "ok")
diff --git a/banyand/stream/tstable.go b/banyand/stream/tstable.go
index f2a48ea0f..0965dc396 100644
--- a/banyand/stream/tstable.go
+++ b/banyand/stream/tstable.go
@@ -245,7 +245,8 @@ func initTSTableWithLease(fileSystem fs.FileSystem, rootPath string, p common.Po
indexMetrics = tst.metrics.indexMetrics
}
if initIndex {
- index, err := newElementIndex(context.TODO(), rootPath, option.elementIndexFlushTimeout.Nanoseconds()/int64(time.Second), indexMetrics, lease)
+ index, err := newElementIndex(context.TODO(), rootPath, option.elementIndexFlushTimeout.Nanoseconds()/int64(time.Second),
+ indexMetrics, p.SegLabelValues(), lease)
if err != nil {
return nil, 0, err
}
diff --git a/banyand/stream/tstable_test.go b/banyand/stream/tstable_test.go
index e698343a2..f02fcc3a5 100644
--- a/banyand/stream/tstable_test.go
+++ b/banyand/stream/tstable_test.go
@@ -91,7 +91,7 @@ func Test_tsTable_mustAddElements(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpPath, _ := test.Space(require.New(t))
- index, _ := newElementIndex(context.TODO(), tmpPath, 0, nil, newTestRootLease(t, tmpPath))
+ index, _ := newElementIndex(context.TODO(), tmpPath, 0, nil, nil, newTestRootLease(t, tmpPath))
tst := &tsTable{
index: index,
loopCloser: run.NewCloser(2),
@@ -230,7 +230,7 @@ func Test_tstIter(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpPath, defFn := test.Space(require.New(t))
- index, _ := newElementIndex(context.TODO(), tmpPath, 0, nil, newTestRootLease(t, tmpPath))
+ index, _ := newElementIndex(context.TODO(), tmpPath, 0, nil, nil, newTestRootLease(t, tmpPath))
defer defFn()
tst := &tsTable{
index: index,
diff --git a/docs/design/0.12.0/README.md b/docs/design/0.12.0/README.md
index 70a99a99d..ab7dcf85c 100644
--- a/docs/design/0.12.0/README.md
+++ b/docs/design/0.12.0/README.md
@@ -2,6 +2,7 @@
- [Tag aggregation and time bucketing in the measure query engine](tag-aggregation/README.md)
- [Self-stored logs](self-stored-logs/README.md)
+- [Time-range pruning in the native inverted index](native-index-time-pruning/README.md)
The [native inverted-index replacement](../archive/0.12.0/native-inverted-index/README.md)
design shipped and has moved to the archive ahead of the rest of this milestone.
diff --git a/docs/design/0.12.0/native-index-time-pruning/README.md b/docs/design/0.12.0/native-index-time-pruning/README.md
new file mode 100644
index 000000000..d447d5f9d
--- /dev/null
+++ b/docs/design/0.12.0/native-index-time-pruning/README.md
@@ -0,0 +1,465 @@
+# Time-Range Pruning in the Native Inverted Index
+
+Status: **implemented**.
+
+The native inverted index (`pkg/index/native`, `pkg/index/internal/nativeice`) already writes
+everything a time-range query needs: per-segment time bounds and a multi-precision `_timestamp`
+term trie. No query reads either. Every query applies its time range per document, after
+decoding that document's stored fields. This design makes the time range narrow candidates
+**before** any stored document is decoded, in two layers:
+
+1. **Segment pruning** — skip a segment whose time bounds miss the query range, and drop the
+ per-document time check for a segment the range fully contains.
+2. **Trie candidates** — for a segment the range only overlaps, turn the range into a few dozen
+ `_timestamp` terms at mixed precision, OR their postings, and intersect the result with the
+ series and filter candidates.
+
+It also fixes a prerequisite defect: the streaming merge writes **zero** time bounds, so a merged
+segment currently reports "no timestamps".
+
+The change needs no format change, no new persisted data, and no schema or API change. It is
+confined to the native query layer and the merge writer.
+
+---
+
+## 1. Problem
+
+### 1.1 What every query does today
+
+All time-scoped native reads go through one of three paths, each checking time per document:
+
+| Entry point | Per-document time check |
+|---|---|
+| `MatchTermsSet`, `MatchAllTermSets`, `MatchField`, `MatchRange` → `projectCandidates` | `query.go:965` |
+| `NewSortCursor` / `NextPage` → `SortCursor.project` | `sort_cursor.go:309` |
+| `memorySegment.MatchTerms` | `owner.go:2421` |
+
+Candidates are built from series postings (`seriesCandidates`, `query.go:897`) and filter terms
+only. Each candidate is then projected, which means:
+- `VisitDocument` decodes its stored fields;
+- the identifier and series ID are copied;
+- the prefix-coded timestamp is decoded.
+
+Only after all that is the document dropped for being out of range. The sort cursor's "no series
+scope" path walks *every* document in the segment (`all = true`).
+
+### 1.2 Why it matters
+
+A segment of the stream element index spans one storage segment (a day by default), while typical
+queries ask for minutes. For a 15-minute window over a day-long segment:
+- about 99% of the candidates of each matching series are decoded and thrown away;
+- in a sort query the same documents also pay for the per-document sort-value lookup and the
+ top-N heap work.
+
+Measured on a related shape (`banyand/stream`, 500k documents, 4 CPUs, `main` at `ca86e9a2`):
+- the native `SortCursor` costs about 0.97 µs wall, 1.4 µs CPU and 16 allocations per candidate;
+- every out-of-range candidate pays that cost in full.
+
+### 1.3 Data the index already has
+
+**Per-document `_timestamp`** (`owner.go:2141-2160`), written for every document with a non-zero
+timestamp:
+- stored;
+- sortable doc value;
+- indexed under **16 terms**, one per precision level.
+
+```go
+for shift := uint(0); shift <= 60; shift += 4 {
+ EncodePrefixCodedInt64Shift(ts, shift) // [0x20+shift][7-bit groups of (ts^signbit)>>shift]
+}
+```
+
+The term at shift `s` stands for the bucket of all timestamps sharing `ts >> s`. The levels run
+from 1 ns (shift 0) through 4.3 s (32), 68.7 s (36), 18.3 min (40), 4.9 h (44) and 3.3 days (48),
+up to decades (60). The shift byte leads, so the terms of one level are contiguous in the
+dictionary and sorted by bucket. This is the classic numeric trie that Lucene's
+`NumericRangeQuery` uses. The native writer emits it to stay compatible with the previous
+release's segments (`nidx-03-series-index.md`, field table).
+
+**Per-segment bounds:**
+- flush segments record `timeMin`/`timeMax` in the segment footer (`footer[36:52]`);
+- the snapshot manifest also records them (`SnapshotSegment.TimeMin/TimeMax`);
+- in memory, `segmentHandle.timeMin/timeMax/hasTime` (`owner.go:362-365`) hold them.
+
+### 1.4 Defect: merged segments lose their bounds
+
+`streamMerger.write` writes `0` for both bounds (`merge_stream.go:432-433`). `MergeStats` carries
+no bounds, and `newSegmentFromMergeFile` (`merge_staging.go:186`) builds the handle with
+`hasTime = false`. Every merged segment therefore claims to have no timestamps, and so does its
+manifest entry once persisted (`owner.go:905`, `989`). Merged segments soon hold most of the data,
+so segment pruning is worthless until this is fixed.
+
+---
+
+## 2. Goals and non-goals
+
+**Goals**
+- G1. A segment the query range misses costs no document decode.
+- G2. In a segment the query range overlaps, out-of-range documents cost no stored-field decode.
+- G3. A segment the query range fully contains pays no per-document time check.
+- G4. Results are identical to today's for every query, including documents without a timestamp,
+ deleted documents, and inclusive or exclusive bounds.
+- G5. Merged segments carry exact time bounds, in the footer and in the manifest.
+- G6. No format change. Segments written by older code, including merged segments with zero
+ bounds, stay readable and return correct results.
+
+**Non-goals**
+- Index-ordered sort that stops early (walking the sort field's terms in order). This design
+ shrinks the sort cursor's candidate set but keeps it a full pass over that set. See §9.
+- Projecting only the top-N rows in the stream vectorized scan (sort-then-fetch). That is a
+ separate fix for the wide-row memory-budget failure.
+- Any change to how callers express time ranges (`index.RangeOpts`, `native.TimeRange`).
+
+---
+
+## 3. Design overview
+
+For each query, segment by segment:
+
+```
+classify(segment, range):
+ bounds unknown → OVERLAP
+ bounds ∩ range = ∅ → DISJOINT — skip the segment
+ bounds ⊆ range → CONTAINED — no range comparison
+ otherwise → OVERLAP
+
+OVERLAP:
+ if segment.trieUsable:
+ timeBits := trieCandidates(segment, range) — exact
+ candidates := candidates ∧ timeBits (or timeBits when "all")
+ project without a time check
+ else:
+ project with today's per-document time check — fallback
+```
+
+| Layer | Removes | Cost |
+|---|---|---|
+| Segment classification | all work on disjoint segments; the per-document check on contained ones | two integer compares per segment |
+| Trie candidates | stored-field decode of out-of-range documents in overlapping segments | at most a few hundred dictionary lookups and posting ORs per segment |
+| Merge bounds (prerequisite) | — | one timestamp decode per live document during merge, on a field the merge already decodes |
+
+---
+
+## 4. Detailed design
+
+### 4.1 Exact time bounds for merged segments
+
+**Where:** `internal/nativeice/merge_sections.go` (`writeStored`), `merge_stream.go` (footer),
+`merge.go` (`MergeStats`, the buffered `MergeSegments`), `native/merge_staging.go`.
+
+1. `writeStored` already walks every stored document of every input, with its fields decoded, and
+ knows whether that document survives (`dropped`). For each **surviving** document, decode the
+ `_timestamp` stored value with `DecodePrefixCodedInt64` and fold it into running `int64`
+ bounds. Skip documents with no `_timestamp`, as the flush path skips `Timestamp == 0`.
+2. Extend `MergeStats` with `TimeMin`, `TimeMax` and `HasTime`. The footer writes the bounds into
+ the existing `footer[36:44]` and `footer[44:52]` slots, encoded as `uint64(int64)` exactly as
+ the flush path does (`owner.go:2141-2149`).
+3. `newSegmentFromMergeFile` sets `handle.timeMin/timeMax/hasTime` from the stats. Persistence
+ (`owner.go:905`, `989`) and lifecycle (`lifecycle.go:126`) already copy the handle's bounds
+ into the manifest, and the manifest/footer consistency check (`nativeice.go:1548`, `1699`) then
+ holds by construction.
+4. The buffered `MergeSegments` path gets the same treatment, so both merge forms agree.
+
+The bounds are computed from the surviving documents rather than combined from the inputs'
+bounds. Combining would work, but inputs written by today's code have no bounds; computing them
+directly means every later merge repairs old merged segments, with no migration.
+
+**Compatibility:** flush segments already write non-zero bounds into the same footer slots, so a
+merged segment with non-zero bounds is not a new case for any reader, including the previous
+release's rollback path (the NIDX-03 rollback proof covered flush segments with bounds).
+
+### 4.2 Classifying segments
+
+**Where:** a new helper in `native/query.go`, used by every entry point in §1.1.
+
+```go
+type timeClass uint8 // timeDisjoint, timeContained, timeOverlap
+
+func classifyTime(h *segmentHandle, r *TimeRange) timeClass
+```
+
+- `r == nil` → `timeContained`: the query has no time restriction, so behaviour is unchanged.
+- `!h.hasTime` → `timeOverlap`. Unknown bounds never prune.
+- Compare in `int64`, converting the handle's `uint64` slots back. If `int64(timeMin) >
+ int64(timeMax)`, the bounds were folded across a sign change, which a pre-epoch timestamp could
+ cause; treat them as unknown (`timeOverlap`).
+- Turn `r` into a closed `int64` interval `[lo, hi]`:
+ - `lo = r.Lower` if inclusive, else `r.Lower + 1`;
+ - `hi = r.Upper` if inclusive, else `r.Upper - 1`;
+ - guard the ±1 against overflow;
+ - if `lo > hi`, the range is empty and every segment is `timeDisjoint`.
+- Then:
+ - `timeMax < lo || timeMin > hi` → `timeDisjoint`;
+ - `lo <= timeMin && timeMax <= hi` → `timeContained`;
+ - otherwise `timeOverlap`.
+
+**Exactness of `timeContained`:** the bounds cover exactly the documents that have a timestamp.
+A contained segment still has to drop documents *without* a timestamp, because today's contract
+excludes them: "a timestamp range excludes documents without a stored native timestamp",
+`QueryScope` doc, `query.go:58`. Projection keeps a cheap `hasTimestamp` check for those but skips
+the range comparison. The `_timestamp` decode itself is still needed because callers consume
+`QueryHit.Timestamp`.
+
+### 4.3 Turning a range into trie candidates
+
+**Where:** `internal/nativeice/numeric.go` (range split) and `native/query.go` (posting union).
+
+**Range split.** Port Lucene's `NumericUtils.splitRange` for a 64-bit value with precision step 4.
+It works on the sign-flipped value (`uint64(v) ^ 1<<63`, the same transform the encoder applies),
+so unsigned order matches signed order.
+
+```go
+// SplitSortableRange covers the closed sortable interval [lo, hi] with
+// the fewest trie buckets and emits each run of consecutive buckets as
+// (shift, firstBucket, lastBucket), in sortable units at that shift.
+func SplitSortableRange(lo, hi uint64, emit func(shift uint, first, last uint64))
+
+for shift := uint(0); ; shift += 4 {
+ diff := uint64(1) << (shift + 4) // width of the next level's bucket
+ mask := uint64(0xF) << shift // this level's digit
+ hasLower := lo&mask != 0 // lo is not aligned to the next level
+ hasUpper := hi&mask != mask // hi does not end the next level's bucket
+ nextLo := lo; if hasLower { nextLo += diff }; nextLo &^= mask
+ nextHi := hi; if hasUpper { nextHi -= diff }; nextHi &^= mask
+ if shift+4 >= 64 || nextLo > nextHi || nextLo < lo || nextHi > hi {
+ emit(shift, lo>>shift, hi>>shift) // the remaining middle, at this level
+ return
+ }
+ if hasLower { emit(shift, lo>>shift, (lo|mask)>>shift) }
+ if hasUpper { emit(shift, (hi&^mask)>>shift, hi>>shift) }
+ lo, hi = nextLo, nextHi
+}
+```
+
+Each emitted run spans at most 16 buckets at its level. The cover therefore has at most
+`2·15·15 + 16 = 466` terms. For realistic windows it has a few dozen: a 15-minute window is about
+12 buckets at shift 36 plus at most 15 at each finer level on each edge.
+
+**Encoding.** Add `encodePrefixCodedSortableShift(sortable, shift)`, the existing encoder minus
+its sign flip. Each emitted run `(shift, first, last)` becomes the dictionary interval
+`[enc(first, shift), enc(last, shift)]`. Dictionary iterators take an **exclusive** end
+(`dictionary_iterator.go:120`), so pass `enc(last+1, shift)`. When `last` is the largest bucket
+at that level, pass the first key of the next shift level instead: `[]byte{0x20 + shift + 1}`,
+or nil above shift 60.
+
+**Posting union.**
+
+```go
+func trieCandidates(ctx context.Context, s *memorySegment, r *TimeRange) (*roaring.Bitmap, error)
+```
+
+- For each run, iterate the `_timestamp` dictionary over the interval and OR each term's posting
+ (the reader's `TermPostingBitmap`, or the iterator's posting accessor) into one bitmap.
+- Check `ctx` between runs.
+- The bitmap is **exact**. Every timestamped document has exactly one term at each level. The
+ cover partitions `[lo, hi]` into disjoint buckets, so a document is in the union if and only if
+ its timestamp is in `[lo, hi]`. Documents without a timestamp have no `_timestamp` terms and
+ are excluded, which matches today's contract.
+- Deleted documents can appear in the bitmap. Deletion is still applied where it is today (the
+ `segment.deleted` check in projection and in `SortCursor.project`).
+
+**Cost.** Every document in range appears in exactly one posting of the cover. So building the
+bitmap costs O(dictionary lookups in the cover + documents in range) in roaring word operations,
+against today's O(candidates) stored-field decodes. The term count is fixed, and no `MaxTerms`
+expansion limit applies, because the cover never iterates exact timestamps.
+
+### 4.4 Checking that a segment has the trie
+
+The trie is correct only if a segment actually carries the coarse terms. Native and NIDX-era
+segments do (`owner.go:2151`), and merges copy every term. Older segments are meant to, by the
+compatibility contract, but a segment missing its coarse terms would silently **lose** documents
+under §4.3. To guard against that, every segment handle gets a lazily computed, cached flag:
+
+```go
+trieOnce sync.Once
+trieUsable bool
+```
+
+- `trieUsable` is true when the segment has no `_timestamp` field.
+- Otherwise it is true only if the `_timestamp` dictionary holds at least one term at shift 4
+ **and** at least one at shift 60. That is two single-key iterator probes:
+ `[0x24, 0x25)` and `[0x5C, 0x5D)`.
+- Every timestamped document writes both levels, so a segment that has timestamps but misses
+ either level was written without the trie, or with a different precision step.
+
+When `trieUsable` is false, the segment takes the fallback path: today's per-document check. A
+counter records how often that happens (§6), so no fallback can go unseen.
+
+### 4.5 Integration per entry point
+
+**`projectCandidates` callers** (`MatchTermsSet`, `MatchAllTermSets`, `MatchField`,
+`MatchRange`). Per segment, before projection:
+- `timeDisjoint` → `continue`. Its candidates do not count toward `MaxCandidates`.
+- `timeContained` → project with the contained flag set, i.e. the `hasTimestamp` check only.
+- `timeOverlap` with a usable trie → `candidates.And(trie)`, then project with the contained flag.
+- `timeOverlap` without a usable trie → project as today.
+
+`projectCandidates` gets a `timeMode` parameter in place of its bare `*TimeRange`. The candidate
+total behind `MaxCandidates` is computed **after** the intersection. This is a deliberate
+behaviour change: a time-narrowed query that used to fail with `ErrQueryLimit` now succeeds.
+Ranges outside the time window should never have counted.
+
+**`NewSortCursor`.**
+- `timeDisjoint` segments are left out of `cursor.segments`.
+- For `timeOverlap` segments with a usable trie:
+ - the series path ANDs the trie into `candidates`;
+ - the `all = true` path replaces "every ordinal" with the trie bitmap (`all = false`,
+ `candidates = trie`).
+- Each `sortCursorSegment` records whether its candidates are exact in time. `project` then skips
+ the range comparison for them (`sort_cursor.go:309`) but keeps the `hasTimestamp` requirement.
+
+**`memorySegment.MatchTerms`** (`owner.go:2361`). The same classification applies.
+`timeDisjoint` returns an empty result, and overlap intersects the term postings with the trie --
+subject to the same `trieMinCandidates` threshold as every other entry point (§4.6): the posting
+being narrowed is the candidate set here, so a point lookup whose posting is small is not worth
+narrowing either.
+
+**Not affected:**
+- `FilterTermsSet` and `FilterRange` already reject `Scope.TimeRange`.
+- `ProjectHit` and `ProjectSortValue` act on already-selected hits.
+
+### 4.6 When to build the trie bitmap
+
+When the non-time candidates are already tiny, decoding them can cost less than OR-ing the
+cover's postings. For example, a point lookup by series with two hits in a long range.
+
+Rule: build the trie bitmap only when `candidates.GetCardinality() >= trieMinCandidates`;
+otherwise use the per-document check. For the `all` path the trie is always built. The threshold
+is a package constant, chosen by the benchmark in §7.2: 128, the crossover
+`BenchmarkPointLookupCrossover` measured between a 64-candidate posting (per-document still wins)
+and a 128-candidate one (the trie wins decisively). It is not a user-facing flag.
+
+---
+
+## 5. Correctness argument
+
+| Concern | Why results are unchanged |
+|---|---|
+| Disjoint pruning | Bounds are exact over the timestamped documents (§4.1, flush path), so a disjoint segment has no in-range document. Untimestamped documents are excluded by contract anyway. Unknown bounds never prune. |
+| Contained skip | Every timestamped document is in range; the `hasTimestamp` check still removes untimestamped ones. |
+| Trie bitmap | Exact cover of `[lo, hi]` (§4.3); usable only where both probe levels exist (§4.4). |
+| Exclusive bounds | Normalised once to a closed `int64` interval with overflow guards; an empty interval prunes everything. |
+| Deletions | Unchanged: still applied at projection and in the sort cursor. |
+| Older segments | Zero bounds read as unknown, so no pruning. A missing trie falls back. Both heal on the next merge (§4.1). |
+| Negative or mixed-sign timestamps | Trie order is sign-correct (sign-flipped sortable). Mixed-sign `uint64` bounds are detected and treated as unknown (§4.2). |
+| Concurrency | Classification reads immutable handle fields. The `trieOnce` probe is per immutable handle and reads only the pinned reader. |
+
+---
+
+## 6. Observability
+
+`pkg/index/metrics` gets per-owner counters, labelled like the existing native metrics:
+
+- `native_time_segments_total{class="disjoint|contained|overlap_trie|overlap_fallback"}`
+- `native_time_candidates_pruned_total`: candidates removed by the trie intersection, i.e.
+ cardinality before minus after.
+
+A non-zero `overlap_fallback` on a cluster whose data was written by native code points to a bug.
+On a cluster with segments from older writers it is expected, and should fall as those segments
+are merged.
+
+---
+
+## 7. Validation
+
+### 7.1 Tests
+
+**`nativeice` unit tests**
+- `SplitSortableRange` against brute force:
+ - property test over random `[lo, hi]` pairs (fixed seed, about 10k cases), including
+ `lo == hi`, aligned and unaligned edges, `MinInt64`, `MaxInt64`, and ranges spanning zero;
+ - every value in the range lies in exactly one emitted bucket, and no value outside does;
+ - the term count stays at or below 466.
+- The encoder: `encodePrefixCodedSortableShift(uint64(v)^1<<63, s)` equals
+ `EncodePrefixCodedInt64Shift(v, s)` for all 16 shifts.
+
+**Merge tests**
+- Footer and manifest bounds equal the exact min/max over surviving documents.
+- Documents dropped by deletion or `Drop` are excluded.
+- All-deleted or untimestamped input yields `HasTime == false` and zero slots.
+- Streaming and buffered merge agree.
+- Merging an input with zero bounds (today's merged output) yields exact bounds.
+
+**Native query tests**, for each entry point in §4.5:
+- Results equal those from the per-document path (forced on through a test hook) over random
+ documents, random windows (empty, point, inclusive and exclusive edges, full span, disjoint)
+ and random deletions.
+- Run over each kind of segment:
+ - flush;
+ - persisted-and-promoted;
+ - streaming-merged;
+ - external-received;
+ - a segment written without coarse terms, built through the internal encoder by emitting only
+ shift-0 terms. This must take the fallback path and still match.
+- A disjoint segment's reader is never asked to visit a document: a test hook counts
+ `VisitDocument` calls.
+- `MaxCandidates` is checked after intersection, and a time-narrowed query that used to fail
+ now passes.
+
+**Integration:** the existing stream, measure and trace standalone and distributed suites, plus
+the stream element-index golden cases, all unchanged. Per the repository rule, the tests must
+not reference the retired legacy library.
+
+### 7.2 Benchmarks
+
+The benchmarks are short `go test -bench` runs pinned to 4 CPUs, reporting wall time, process
+CPU (rusage) per operation, B/op and allocs/op. No long soak is part of this design.
+
+The fixture extends `banyand/stream`'s index-order benchmark (500k documents over a day-scale
+segment, 100 series):
+
+| Axis | Values |
+|---|---|
+| Window, as a share of the segment's span | 0.1%, 1%, 10%, 100% |
+| Query | filter (`EQ` on an indexed tag, per series) · index sort `LIMIT 20` · series point lookup |
+| Segment state | flushed only · after a full merge |
+| Build | `main` · this design |
+
+**Gates:**
+- **G1/G2:** at a 1% window, filter and sort CPU per operation drop by at least 5×, and
+ allocated bytes drop by a similar factor.
+- **No regression:** at a 100% window, and for the point lookup, CPU and B/op stay within ±5%
+ of `main`.
+- **Merged state:** after a merge, the 1% window shows the same gain as before it. This proves
+ the fix in §4.1.
+- `trieMinCandidates` is set at the crossover where the point lookup stops regressing.
+
+---
+
+## 8. Rollout and compatibility
+
+- **On-disk format:** unchanged. Merged segments start filling two footer slots that flush
+ segments already fill. Manifests are unchanged in shape.
+- **Mixed or rolled-back binaries:** an older binary ignores the bounds for queries and keeps
+ checking per document. A newer binary treats older merged segments as unknown-bounds. Both
+ return correct results.
+- **No flag:** the optimisation is exact. The fallback path and the
+ `native_time_segments_total{class="overlap_fallback"}` counter cover segments that cannot take
+ it.
+- **Delivery:** one PR with three commits: (1) merge bounds plus its tests; (2) range split,
+ `trieUsable` and `trieCandidates`, plus their unit tests; (3) entry-point integration, metrics
+ and benchmarks.
+
+---
+
+## 9. Follow-ups, out of scope
+
+1. **Index sort that stops early.** Walk the sort field's dictionary in sort order, intersect
+ each term's posting with the (now time-narrowed) candidates, and stop after N hits. Combined
+ with this design, an index-ordered `LIMIT N` over a short window becomes a short posting walk
+ instead of a candidate scan. Needs its own benchmark against the vectorized block sort.
+2. **Vectorized sort-then-fetch for stream.** Order on the sort key plus row references, then
+ decode the projection for only the top N. It fixes the `LIMIT 20` query that fails today with
+ `vectorized query memory budget exceeded` on wide rows.
+3. **Faster projection.** With time handled by the candidate bitmap, contained and overlap-trie
+ projection still decode `_timestamp` for `QueryHit.Timestamp`. Reading it from the
+ `_timestamp` doc value instead of the stored document would avoid the stored walk for callers
+ that need only identifier, series and timestamp.
+
+## 10. Open questions
+
+1. Should `trieMinCandidates` instead compare candidates against the segment's document count in
+ range (a selectivity ratio), rather than an absolute count? Decide from the §7.2 data.
+2. Do any production segments predate the 0..60/step-4 trie, i.e. legacy-written element indexes
+ never migrated? The `overlap_fallback` counter answers this in the field; no code depends on
+ the answer.
diff --git a/pkg/index/internal/nativeice/encode.go b/pkg/index/internal/nativeice/encode.go
index 67f0d90ec..80397d7f8 100644
--- a/pkg/index/internal/nativeice/encode.go
+++ b/pkg/index/internal/nativeice/encode.go
@@ -36,6 +36,14 @@ import (
// values and a selection resolves it like any other exact term.
const identifierField = "_id"
+// timestampField is the name ICE v3 reserves for a document's timestamp: the
+// shift-zero prefix-coded value Encode stores, plus the multi-precision terms
+// that let a query narrow a time range before decoding any document. The
+// merge reads it back to compute the merged segment's time bounds, which is
+// what lets a query skip or cheaply narrow a segment it would otherwise scan
+// in full.
+const timestampField = "_timestamp"
+
// ErrInvalidGeneration is the sentinel reported when a caller asks Encode to
// write a generation the ICE v3 grammar has no representation for: a document
// carrying no identifier, or a field carrying no name. It is deliberately
diff --git a/pkg/index/internal/nativeice/merge.go b/pkg/index/internal/nativeice/merge.go
index e3c6c9b6f..7bb6a1609 100644
--- a/pkg/index/internal/nativeice/merge.go
+++ b/pkg/index/internal/nativeice/merge.go
@@ -60,6 +60,15 @@ type MergeStats struct {
DocumentCount uint64
// Size is the number of segment bytes written.
Size uint64
+ // TimeMin and TimeMax are the merged segment's exact time bounds, encoded
+ // as uint64(int64) exactly as the footer stores them. They cover the
+ // surviving documents that carry a timestamp; HasTime reports whether any
+ // did. A merge of input whose own bounds are unknown still produces exact
+ // bounds, because they are folded from the surviving documents rather than
+ // combined from the inputs.
+ TimeMin uint64
+ TimeMax uint64
+ HasTime bool
}
// MergeSegments merges native single-segment readers into an in-memory
@@ -170,7 +179,13 @@ func MergeSegmentsTo(ctx context.Context, inputs []MergeInput, output io.Writer,
if writeErr := merger.write(); writeErr != nil {
return MergeStats{}, writeErr
}
- return MergeStats{DocumentCount: merger.documentCount, Size: merger.output.written}, nil
+ return MergeStats{
+ DocumentCount: merger.documentCount,
+ Size: merger.output.written,
+ TimeMin: merger.timeMin,
+ TimeMax: merger.timeMax,
+ HasTime: merger.hasTime,
+ }, nil
}
// countingWriter tracks the running segment offset every section records.
@@ -219,6 +234,9 @@ func acquireStreamMerger(ctx context.Context, output io.Writer, spillPrefix stri
merger.spill = spillFactory{prefix: spillPrefix}
merger.inputs, merger.fields, merger.fieldIDs = nil, nil, nil
merger.documentCount, merger.storedIndex = 0, 0
+ // The bounds are folded per merge, so a pooled merger must not carry the
+ // previous merge's.
+ merger.timeMin, merger.timeMax, merger.hasTime = 0, 0, false
return merger
}
diff --git a/pkg/index/internal/nativeice/merge_sections.go b/pkg/index/internal/nativeice/merge_sections.go
index df478c2a1..b1ab8e359 100644
--- a/pkg/index/internal/nativeice/merge_sections.go
+++ b/pkg/index/internal/nativeice/merge_sections.go
@@ -64,6 +64,7 @@ func (m *streamMerger) writeStored() error {
// which stays loaded for this whole document callback.
var identifier []byte
hasIdentifier := false
+ var timestampValue []byte
m.storedValues = append(m.storedValues[:0], storedValue{name: identifierField})
for _, field := range fields {
if field.name == identifierField {
@@ -72,6 +73,11 @@ func (m *streamMerger) writeStored() error {
identifier = field.value
continue
}
+ if field.name == timestampField {
+ // Remember it for the bounds below; it is written like any
+ // other surviving stored value.
+ timestampValue = field.value
+ }
if dropped {
continue
}
@@ -86,6 +92,26 @@ func (m *streamMerger) writeStored() error {
if dropped {
return nil
}
+ // Fold this survivor's timestamp into the merged bounds. The flush
+ // path treats a zero timestamp as "no timestamp", so the merged
+ // segment does too: bounds describe exactly the documents a time
+ // range can match, and a document without one is never matched.
+ if len(timestampValue) > 0 {
+ timestamp, decodeErr := DecodePrefixCodedInt64(timestampValue)
+ if decodeErr != nil {
+ return fmt.Errorf("physical document %d has an undecodable timestamp: %w", local, decodeErr)
+ }
+ if timestamp != 0 {
+ encoded := uint64(timestamp)
+ if !m.hasTime || encoded < m.timeMin {
+ m.timeMin = encoded
+ }
+ if !m.hasTime || encoded > m.timeMax {
+ m.timeMax = encoded
+ }
+ m.hasTime = true
+ }
+ }
m.storedValues[0].value = identifier
binary.BigEndian.PutUint64(offsetBytes[:], uint64(len(m.decodedChunk)))
if _, writeErr := offsets.Write(offsetBytes[:]); writeErr != nil {
diff --git a/pkg/index/internal/nativeice/merge_stream.go b/pkg/index/internal/nativeice/merge_stream.go
index e591980b5..5cbfd0b04 100644
--- a/pkg/index/internal/nativeice/merge_stream.go
+++ b/pkg/index/internal/nativeice/merge_stream.go
@@ -76,6 +76,14 @@ type streamMerger struct {
documentCount uint64
storedIndex uint64
+ // Time bounds over the surviving timestamped documents, in the same
+ // uint64(int64) form the flush path writes into the footer. hasTime is
+ // false when no surviving document carried a timestamp, which leaves the
+ // footer slots at zero and marks the segment as having no time bounds.
+ timeMin uint64
+ timeMax uint64
+ hasTime bool
+
// Reusable section buffers: the merge keeps one steady working set
// instead of allocating per document or per term.
storedValues []storedValue
@@ -429,8 +437,12 @@ func (m *streamMerger) write() error {
binary.BigEndian.PutUint64(footer[16:24], fieldsIndexOffset)
binary.BigEndian.PutUint64(footer[24:32], docValueOffset)
binary.BigEndian.PutUint32(footer[32:36], nativeICEChunkModeV1)
- binary.BigEndian.PutUint64(footer[36:44], 0)
- binary.BigEndian.PutUint64(footer[44:52], 0)
+ // The time bounds a flush segment already writes into these slots. Writing
+ // them here as well is what lets a query skip or narrow a merged segment
+ // instead of decoding every candidate it holds; leaving them at zero is what
+ // previously made every merged segment report "no timestamps".
+ binary.BigEndian.PutUint64(footer[36:44], m.timeMin)
+ binary.BigEndian.PutUint64(footer[44:52], m.timeMax)
binary.BigEndian.PutUint32(footer[52:56], segmentVersion)
_, writeErr := m.output.Write(footer)
return writeErr
diff --git a/pkg/index/internal/nativeice/merge_time_bounds_test.go b/pkg/index/internal/nativeice/merge_time_bounds_test.go
new file mode 100644
index 000000000..71186d22a
--- /dev/null
+++ b/pkg/index/internal/nativeice/merge_time_bounds_test.go
@@ -0,0 +1,317 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nativeice
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "testing"
+
+ roaringpkg "github.com/RoaringBitmap/roaring"
+)
+
+// timestampDocument builds a document carrying a stored _timestamp exactly the
+// way the owner writes one, so a merge has the same value to fold.
+func timestampDocument(identifier string, timestamp int64) EncodeDocument {
+ return EncodeDocument{
+ Identifier: []byte(identifier),
+ Fields: []EncodeField{{
+ Name: timestampField,
+ Value: EncodePrefixCodedInt64(timestamp),
+ Terms: timestampTerms(timestamp), Store: true, Index: true, Sort: true,
+ }},
+ }
+}
+
+// timestampTerms mirrors the owner's multi-precision _timestamp term trie.
+func timestampTerms(timestamp int64) []EncodeTerm {
+ terms := make([]EncodeTerm, 0, 16)
+ for shift := uint(0); shift <= 60; shift += 4 {
+ terms = append(terms, EncodeTerm{Value: EncodePrefixCodedInt64Shift(timestamp, shift), Frequency: 1})
+ }
+ return terms
+}
+
+// footerBounds reads the two time slots back out of an encoded segment's
+// footer, which is what a reopened reader would see.
+func footerBounds(t *testing.T, payload []byte) (uint64, uint64) {
+ t.Helper()
+ if len(payload) < segmentFooterLength {
+ t.Fatalf("payload %d bytes is shorter than a footer", len(payload))
+ }
+ footer := payload[len(payload)-segmentFooterLength:]
+ return binary.BigEndian.Uint64(footer[36:44]), binary.BigEndian.Uint64(footer[44:52])
+}
+
+// encodeInputs writes one segment per generation and returns their readers.
+func encodeInputs(t *testing.T, generations ...Generation) []*Reader {
+ t.Helper()
+ readers := make([]*Reader, 0, len(generations))
+ for index := range generations {
+ path := t.TempDir()
+ if encodeErr := Encode(path, generations[index]); encodeErr != nil {
+ t.Fatal(encodeErr)
+ }
+ reader, openErr := OpenStrict(path)
+ if openErr != nil {
+ t.Fatal(openErr)
+ }
+ t.Cleanup(func() { _ = reader.Close() })
+ readers = append(readers, reader)
+ }
+ return readers
+}
+
+func TestMergeComputesExactTimeBounds(t *testing.T) {
+ readers := encodeInputs(t,
+ Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("a", 1_700_000_000_000_000_000),
+ timestampDocument("b", 1_800_000_000_000_000_000),
+ }},
+ Generation{SegmentID: 2, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("c", 1_750_000_000_000_000_000),
+ }},
+ )
+ inputs := []MergeInput{
+ {Reader: readers[0], IndexedFields: []string{timestampField}},
+ {Reader: readers[1], IndexedFields: []string{timestampField}},
+ }
+
+ var output bytes.Buffer
+ stats, mergeErr := MergeSegmentsTo(context.Background(), inputs, &output, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+
+ const wantMin = int64(1_700_000_000_000_000_000)
+ const wantMax = int64(1_800_000_000_000_000_000)
+ if !stats.HasTime {
+ t.Fatal("merged segment reported no time bounds")
+ }
+ if int64(stats.TimeMin) != wantMin {
+ t.Errorf("TimeMin = %d, want %d", int64(stats.TimeMin), wantMin)
+ }
+ if int64(stats.TimeMax) != wantMax {
+ t.Errorf("TimeMax = %d, want %d", int64(stats.TimeMax), wantMax)
+ }
+
+ // The footer must carry the same values, or a reopened segment would read
+ // them back as zero and lose the bounds the merge computed.
+ footerMin, footerMax := footerBounds(t, output.Bytes())
+ if footerMin != stats.TimeMin || footerMax != stats.TimeMax {
+ t.Errorf("footer bounds = (%d, %d), want (%d, %d)", footerMin, footerMax, stats.TimeMin, stats.TimeMax)
+ }
+}
+
+func TestMergeTimeBoundsExcludeDroppedDocuments(t *testing.T) {
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("keep-low", 1_000),
+ timestampDocument("drop-min", 2_000),
+ timestampDocument("drop-max", 9_000),
+ timestampDocument("keep-high", 5_000),
+ }})
+
+ // Drop the two extreme documents. The bounds must describe only what
+ // survives, otherwise a query could skip a segment that still holds an
+ // in-range document.
+ drop := roaringpkg.New()
+ drop.Add(1)
+ drop.Add(2)
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], Drop: drop, IndexedFields: []string{timestampField}}}, &bytes.Buffer{}, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+ if int64(stats.TimeMin) != 1_000 || int64(stats.TimeMax) != 5_000 {
+ t.Errorf("bounds = (%d, %d), want (1000, 5000) over survivors only",
+ int64(stats.TimeMin), int64(stats.TimeMax))
+ }
+}
+
+func TestMergeTimeBoundsExcludeDeletedDocuments(t *testing.T) {
+ deleted := timestampDocument("gone", 1)
+ deleted.Deleted = true
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ deleted,
+ timestampDocument("alive", 4_200),
+ }})
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], IndexedFields: []string{timestampField}}}, &bytes.Buffer{}, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+ if int64(stats.TimeMin) != 4_200 || int64(stats.TimeMax) != 4_200 {
+ t.Errorf("bounds = (%d, %d), want (4200, 4200); a deleted document must not widen them",
+ int64(stats.TimeMin), int64(stats.TimeMax))
+ }
+}
+
+func TestMergeWithoutTimestampsReportsNoTime(t *testing.T) {
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ {Identifier: []byte("a"), Fields: []EncodeField{
+ {Name: "tag", Value: []byte("v"), Terms: []EncodeTerm{{Value: []byte("t")}}, Index: true, Store: true},
+ }},
+ }})
+ var output bytes.Buffer
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], IndexedFields: []string{"tag"}}}, &output, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+ if stats.HasTime {
+ t.Errorf("HasTime = true for a segment with no timestamps (bounds %d, %d)", stats.TimeMin, stats.TimeMax)
+ }
+ if stats.TimeMin != 0 || stats.TimeMax != 0 {
+ t.Errorf("bounds = (%d, %d), want zero slots", stats.TimeMin, stats.TimeMax)
+ }
+ footerMin, footerMax := footerBounds(t, output.Bytes())
+ if footerMin != 0 || footerMax != 0 {
+ t.Errorf("footer bounds = (%d, %d), want zero", footerMin, footerMax)
+ }
+}
+
+func TestMergeZeroTimestampIsNotTime(t *testing.T) {
+ // The flush path skips a zero timestamp entirely, so a merged segment must
+ // agree: treating it as a bound would claim documents a range can never
+ // match.
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("zero", 0),
+ }})
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], IndexedFields: []string{timestampField}}}, &bytes.Buffer{}, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+ if stats.HasTime {
+ t.Errorf("HasTime = true for a lone zero timestamp (bounds %d, %d)", stats.TimeMin, stats.TimeMax)
+ }
+}
+
+func TestMergeRepairsInputsWithoutBounds(t *testing.T) {
+ // Inputs written before merged segments carried bounds have zero slots. A
+ // merge must still produce exact bounds rather than inheriting the gap,
+ // which is what makes old merged segments heal without a migration.
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("a", 10),
+ timestampDocument("b", 20),
+ }})
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], IndexedFields: []string{timestampField}}}, &bytes.Buffer{}, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+ if int64(stats.TimeMin) != 10 || int64(stats.TimeMax) != 20 {
+ t.Errorf("bounds = (%d, %d), want (10, 20)", int64(stats.TimeMin), int64(stats.TimeMax))
+ }
+}
+
+func TestMergeFormsAgreeOnTimeBounds(t *testing.T) {
+ readers := encodeInputs(t,
+ Generation{SegmentID: 1, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("a", 300), timestampDocument("b", 100),
+ }},
+ Generation{SegmentID: 2, SnapshotID: 1, Documents: []EncodeDocument{
+ timestampDocument("c", 200),
+ }},
+ )
+ inputs := []MergeInput{
+ {Reader: readers[0], IndexedFields: []string{timestampField}},
+ {Reader: readers[1], IndexedFields: []string{timestampField}},
+ }
+ streamed, streamErr := MergeSegmentsTo(context.Background(), inputs, &bytes.Buffer{}, "")
+ if streamErr != nil {
+ t.Fatal(streamErr)
+ }
+ buffered, bufferedErr := MergeSegments(context.Background(), inputs)
+ if bufferedErr != nil {
+ t.Fatal(bufferedErr)
+ }
+ // The buffered form returns a payload only; reopen it so both forms are
+ // compared through the same footer.
+ reader, openErr := OpenSegment(buffered.Payload)
+ if openErr != nil {
+ t.Fatal(openErr)
+ }
+ defer func() { _ = reader.Close() }()
+ bufferedMin, bufferedMax := footerBounds(t, buffered.Payload)
+ if streamed.TimeMin != bufferedMin || streamed.TimeMax != bufferedMax {
+ t.Errorf("streamed bounds (%d, %d) differ from buffered (%d, %d)",
+ streamed.TimeMin, streamed.TimeMax, bufferedMin, bufferedMax)
+ }
+}
+
+func TestMergeTimeBoundsMatchTheFlushPathAcrossASignChange(t *testing.T) {
+ // The bounds are the uint64 slots the footer holds, and the flush path
+ // folds them with an unsigned comparison (owner.go). A set spanning zero
+ // therefore folds to a pair whose int64 reading is inverted, that is
+ // int64(min) > int64(max).
+ //
+ // The merge must fold them exactly the same way, because the merged bytes
+ // have to stay identical to what EncodeSegment writes for the same
+ // documents. It deliberately does not "fix" the ordering: a query reading
+ // these bounds detects int64(min) > int64(max) and treats the segment as
+ // having unknown bounds, which skips pruning rather than pruning wrongly.
+ documents := []EncodeDocument{
+ timestampDocument("neg", -5_000),
+ timestampDocument("pos", 5_000),
+ }
+ readers := encodeInputs(t, Generation{SegmentID: 1, SnapshotID: 1, Documents: documents})
+
+ var output bytes.Buffer
+ stats, mergeErr := MergeSegmentsTo(context.Background(),
+ []MergeInput{{Reader: readers[0], IndexedFields: []string{timestampField}}}, &output, "")
+ if mergeErr != nil {
+ t.Fatal(mergeErr)
+ }
+
+ // Fold the reference way to establish what the footer must contain.
+ var wantMin, wantMax uint64
+ var hasTime bool
+ for _, document := range documents {
+ for _, field := range document.Fields {
+ if field.Name != timestampField {
+ continue
+ }
+ timestamp, decodeErr := DecodePrefixCodedInt64(field.Value)
+ if decodeErr != nil {
+ t.Fatal(decodeErr)
+ }
+ encoded := uint64(timestamp)
+ if !hasTime || encoded < wantMin {
+ wantMin = encoded
+ }
+ if !hasTime || encoded > wantMax {
+ wantMax = encoded
+ }
+ hasTime = true
+ }
+ }
+ if !hasTime {
+ t.Fatal("test premise broken: no timestamp folded")
+ }
+ // The premise is that the unsigned fold really does invert here.
+ if int64(wantMin) <= int64(wantMax) {
+ t.Fatalf("test premise broken: bounds (%d, %d) do not invert", wantMin, wantMax)
+ }
+ if stats.TimeMin != wantMin || stats.TimeMax != wantMax {
+ t.Errorf("merge bounds = (%d, %d), want the flush path's (%d, %d)",
+ stats.TimeMin, stats.TimeMax, wantMin, wantMax)
+ }
+ if footerMin, footerMax := footerBounds(t, output.Bytes()); footerMin != wantMin || footerMax != wantMax {
+ t.Errorf("footer bounds = (%d, %d), want (%d, %d)", footerMin, footerMax, wantMin, wantMax)
+ }
+}
diff --git a/pkg/index/internal/nativeice/nativeice.go b/pkg/index/internal/nativeice/nativeice.go
index a7a94980c..05ec8991b 100644
--- a/pkg/index/internal/nativeice/nativeice.go
+++ b/pkg/index/internal/nativeice/nativeice.go
@@ -116,6 +116,17 @@ type Reader struct {
// segmentReaders holds each segment's stored reader once built, read
// without a lock on the per-lookup path.
segmentReaders atomic.Pointer[[]atomic.Pointer[storedSegmentReader]]
+ // visitDocumentCalls counts every VisitDocument call, successful or not.
+ // It costs one uncontended atomic add per call and exists so a test can
+ // prove a code path decoded zero documents -- for example, that a segment
+ // a time range disjoints from is never asked to visit one.
+ visitDocumentCalls atomic.Int64
+}
+
+// VisitDocumentCalls returns how many times VisitDocument has been called on
+// this Reader. It is test instrumentation: production code never reads it.
+func (r *Reader) VisitDocumentCalls() int64 {
+ return r.visitDocumentCalls.Load()
}
// SnapshotSegment is the immutable metadata a snapshot manifest records for
@@ -976,6 +987,7 @@ func (r *Reader) storedReader(segmentIndex int) (*storedSegmentReader, error) {
// not hold the files open, so if a Close releases them while it reads it
// reports ErrReaderClosed. Its callback may close the Reader.
func (r *Reader) VisitDocument(number uint64, visit func(StoredDocument) error) error {
+ r.visitDocumentCalls.Add(1)
if useErr := r.check(); useErr != nil {
return useErr
}
diff --git a/pkg/index/internal/nativeice/numeric.go b/pkg/index/internal/nativeice/numeric.go
index ee362dbfb..2e0a4fc66 100644
--- a/pkg/index/internal/nativeice/numeric.go
+++ b/pkg/index/internal/nativeice/numeric.go
@@ -30,13 +30,26 @@ func EncodePrefixCodedInt64(value int64) []byte {
// precision shift. It matches the legacy numeric/date term format, where each
// byte carries seven sortable bits and the header records the shift.
func EncodePrefixCodedInt64Shift(value int64, shift uint) []byte {
+ if shift > 63 {
+ return nil
+ }
+ return encodePrefixCodedSortableShift(uint64(value)^0x8000000000000000, shift)
+}
+
+// encodePrefixCodedSortableShift is EncodePrefixCodedInt64Shift without the
+// sign flip: it encodes a value already in sign-flipped sortable order, so
+// unsigned order matches signed order.
+//
+// Callers that hold a sign-flipped value need this rather than re-deriving it,
+// because a range cover is computed in sortable units and then has to be
+// encoded per shift without flipping again.
+func encodePrefixCodedSortableShift(sortable uint64, shift uint) []byte {
if shift > 63 {
return nil
}
nChars := ((63 - shift) / 7) + 1
encoded := make([]byte, nChars+1)
encoded[0] = prefixCodedInt64ShiftStart + byte(shift)
- sortable := uint64(value) ^ 0x8000000000000000
sortable >>= shift
for index := len(encoded) - 1; index > 0; index-- {
encoded[index] = byte(sortable & 0x7f)
@@ -45,6 +58,144 @@ func EncodePrefixCodedInt64Shift(value int64, shift uint) []byte {
return encoded
}
+// sortableInt64 maps a signed int64 into unsigned order that matches signed
+// order, which is the transform every _timestamp term is built on.
+func sortableInt64(value int64) uint64 {
+ return uint64(value) ^ 0x8000000000000000
+}
+
+// sortableRangeBounds is the widest trie cover SplitSortableRange can produce:
+// two full lower and upper staircases of 15 buckets each, plus the 16 buckets
+// of one aligned run at a single level.
+const sortableRangeMaxTerms = 2*15*15 + 16
+
+// TimestampCover runs covers the closed sortable interval [lo, hi] with
+// dictionary term ranges instead of bucket indices, so a caller can iterate a
+// segment's _timestamp dictionary directly.
+type TimestampCover struct {
+ // Start is the first dictionary key in the run, inclusive.
+ Start []byte
+ // End is the exclusive dictionary key one past the run. It is nil only
+ // where the run reaches the top shift, where no larger key exists at this
+ // level.
+ End []byte
+}
+
+// TimestampCoverKeys runs SplitSortableRange over [lo, hi] and turns each
+// emitted bucket run into the dictionary bounds that select it. Keys are in
+// sortable units, so a caller passes sortableInt64 of its own range.
+//
+// The end key is the encoding of the bucket after the run, because dictionary
+// iterators take an exclusive bound. Where the run reaches the largest bucket
+// at its level there is no such key, so the end becomes the first key of the
+// next shift level: the shift byte leads each key, so every key of this level
+// sorts below it. Above the top shift the end is nil, which an iterator reads
+// as unbounded.
+func TimestampCoverKeys(lo, hi uint64) []TimestampCover {
+ runs := make([]struct {
+ shift uint
+ first, last uint64
+ }, 0, 32)
+ SplitSortableRange(lo, hi, func(shift uint, first, last uint64) {
+ runs = append(runs, struct {
+ shift uint
+ first, last uint64
+ }{shift: shift, first: first, last: last})
+ })
+ covers := make([]TimestampCover, 0, len(runs))
+ for _, run := range runs {
+ // A run's buckets are already shifted down, and the encoder shifts
+ // right again, so they have to be shifted back up before encoding.
+ covers = append(covers, TimestampCover{
+ Start: encodePrefixCodedSortableShift(run.first<<run.shift, run.shift),
+ End: coverRunEnd(run.shift, run.last),
+ })
+ }
+ return covers
+}
+
+// coverRunEnd returns the exclusive dictionary bound just past a run that ends
+// at bucket last of the given shift.
+func coverRunEnd(shift uint, last uint64) []byte {
+ if shift >= 60 {
+ // 60 is the highest shift the writer emits, so nothing sorts above it.
+ return nil
+ }
+ if last < maxSortableBucket(shift) {
+ return encodePrefixCodedSortableShift((last+1)<<shift, shift)
+ }
+ // last is the largest bucket this level can express, so bound the run by
+ // the first key of the next level instead.
+ return []byte{prefixCodedInt64ShiftStart + byte(shift) + 1}
+}
+
+// maxSortableBucket is the largest bucket a document's timestamp can fall in
+// at this shift, that is the largest value of ts>>shift.
+//
+// It is the value range rather than the encoder's seven-bits-per-byte width,
+// which is always at least as wide: the encoder's width is
+// 7*ceil((63-shift)/7 + 1) bits against a value range of 64-shift bits. So
+// last+1 always encodes distinctly below this bound, and only a run reaching
+// this bound needs the next-shift-level end key.
+func maxSortableBucket(shift uint) uint64 {
+ if shift >= 64 {
+ return 0
+ }
+ return uint64(1)<<(64-shift) - 1
+}
+
+// SplitSortableRange covers the closed sortable interval [lo, hi] with the
+// fewest possible trie buckets, calling emit once per run of consecutive
+// buckets as (shift, first, last) in sortable units at that shift.
+//
+// It is Lucene's NumericUtils.splitRange specialised to a 64-bit value and the
+// prefix-coded precision step of 4 that the writer emits, and exists so a time
+// range becomes a handful of _timestamp terms instead of a scan. Because the
+// writer indexes every document under one term per level, and this cover
+// partitions [lo, hi] into disjoint buckets, a document is in the union of the
+// emitted runs' postings if and only if its timestamp is in [lo, hi].
+//
+// The cover never exceeds sortableRangeMaxTerms terms, and for a realistic
+// window it is a few dozen: a 15-minute window is about 12 buckets at shift 36
+// plus at most 15 at each finer level on either edge.
+//
+// emit must not retain the shift or bucket values; they are only valid for the
+// duration of the call.
+func SplitSortableRange(lo, hi uint64, emit func(shift uint, first, last uint64)) {
+ if lo > hi {
+ return
+ }
+ for shift := uint(0); ; shift += 4 {
+ diff := uint64(1) << (shift + 4)
+ mask := uint64(0xF) << shift
+ hasLower := lo&mask != 0
+ hasUpper := hi&mask != mask
+ nextLo := lo
+ if hasLower {
+ nextLo += diff
+ }
+ nextLo &^= mask
+ nextHi := hi
+ if hasUpper {
+ nextHi -= diff
+ }
+ nextHi &^= mask
+ if shift+4 >= 64 || nextLo > nextHi || nextLo < lo || nextHi > hi {
+ // Either the next level would not narrow the range, or it would
+ // step outside it, so the remaining middle is emitted whole here.
+ emit(shift, lo>>shift, hi>>shift)
+ return
+ }
+ if hasLower {
+ emit(shift, lo>>shift, (lo|mask)>>shift)
+ }
+ if hasUpper {
+ emit(shift, (hi&^mask)>>shift, hi>>shift)
+ }
+ lo, hi = nextLo, nextHi
+ }
+}
+
// DecodePrefixCodedInt64 decodes the shift-zero prefix-coded signed integer
// used by the established date/time stored-field format. It intentionally
// accepts only shift-zero values: date/time fields are full-precision int64
diff --git a/pkg/index/internal/nativeice/numeric_cover_test.go b/pkg/index/internal/nativeice/numeric_cover_test.go
new file mode 100644
index 000000000..cdec28a4c
--- /dev/null
+++ b/pkg/index/internal/nativeice/numeric_cover_test.go
@@ -0,0 +1,204 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nativeice
+
+import (
+ "bytes"
+ "math/rand"
+ "sort"
+ "testing"
+ "time"
+)
+
+// dictionaryTermsFromCover returns every term a dictionary would yield for the
+// cover, using the same exclusive-end iterator contract the query path uses.
+func dictionaryTermsFromCover(t *testing.T, terms [][]byte, covers []TimestampCover) [][]byte {
+ t.Helper()
+ sorted := make([][]byte, 0, len(terms))
+ sorted = append(sorted, terms...)
+ sort.Slice(sorted, func(left, right int) bool { return bytes.Compare(sorted[left], sorted[right]) < 0 })
+
+ selected := make([][]byte, 0, len(terms))
+ for _, cover := range covers {
+ if len(cover.Start) == 0 {
+ continue
+ }
+ for _, term := range sorted {
+ if bytes.Compare(term, cover.Start) < 0 {
+ continue
+ }
+ if cover.End != nil && bytes.Compare(term, cover.End) >= 0 {
+ break
+ }
+ selected = append(selected, term)
+ }
+ }
+ return selected
+}
+
+// TestTimestampCoverKeysSelectExactlyTheRange builds the real multi-precision
+// term trie for a set of timestamps and checks that a cover's dictionary bounds
+// select exactly the terms of documents in range -- the property the query path
+// depends on for its candidate bitmap to be exact.
+func TestTimestampCoverKeysSelectExactlyTheRange(t *testing.T) {
+ const begin = int64(1_700_000_000_000_000_000)
+ timestamps := []int64{begin}
+ for offset := int64(0); offset <= 4*int64(time.Hour); offset += int64(7 * time.Second) {
+ timestamps = append(timestamps, begin+offset)
+ }
+
+ byShift := make([][][]byte, 16)
+ for _, value := range timestamps {
+ for index, shift := 0, uint(0); shift <= 60; shift, index = shift+4, index+1 {
+ byShift[index] = append(byShift[index], EncodePrefixCodedInt64Shift(value, shift))
+ }
+ }
+
+ source := rand.New(rand.NewSource(31)) //nolint:gosec // deterministic test fixture, not security
+ windows := [][2]int64{
+ {begin, begin},
+ {begin, begin + 1},
+ {begin, begin + int64(15*time.Minute)},
+ {begin, begin + int64(time.Hour)},
+ {begin - 1, begin + 4*int64(time.Hour)},
+ {begin + 1, begin + int64(30*time.Minute) - 1},
+ {begin + int64(7*time.Minute), begin + int64(22*time.Minute)},
+ }
+ for iteration := 0; iteration < 40; iteration++ {
+ low := timestamps[source.Intn(len(timestamps))]
+ high := timestamps[source.Intn(len(timestamps))]
+ if low > high {
+ low, high = high, low
+ }
+ windows = append(windows, [2]int64{low, high})
+ }
+
+ for _, window := range windows {
+ covers := TimestampCoverKeys(sortableInt64(window[0]), sortableInt64(window[1]))
+ if len(covers) == 0 {
+ t.Fatalf("cover of [%d, %d] produced no runs", window[0], window[1])
+ }
+ // A cover emits runs at mixed shifts, so a document joins the union if
+ // ANY emitted run selects its term at that run's shift. That is exactly
+ // how the query path builds its candidate bitmap, and because the runs
+ // partition [lo, hi] the union has to equal the in-range documents.
+ // Union every emitted run's selected terms, then check each timestamp
+ // against it by its term at that run's shift.
+ selected := make(map[string]struct{})
+ for _, cover := range covers {
+ runShift := shiftOf(t, cover.Start)
+ for _, term := range dictionaryTermsFromCover(t, byShift[runShift/4], []TimestampCover{cover}) {
+ selected[string(term)] = struct{}{}
+ }
+ }
+ for valueIndex, value := range timestamps {
+ inRange := value >= window[0] && value <= window[1]
+ coveredByAnyRun := false
+ for shiftIndex := range byShift {
+ if _, present := selected[string(byShift[shiftIndex][valueIndex])]; present {
+ coveredByAnyRun = true
+ break
+ }
+ }
+ if coveredByAnyRun != inRange {
+ t.Fatalf("window [%d, %d]: timestamp %d selected=%v, want %v (covers %d run(s))",
+ window[0], window[1], value, coveredByAnyRun, inRange, len(covers))
+ }
+ }
+ }
+}
+
+// shiftOf recovers the precision shift a term was encoded at from its header.
+func shiftOf(t *testing.T, term []byte) uint {
+ t.Helper()
+ if len(term) == 0 || term[0] < prefixCodedInt64ShiftStart {
+ t.Fatalf("term %x has no shift header", term)
+ }
+ return uint(term[0] - prefixCodedInt64ShiftStart)
+}
+
+func TestTimestampCoverKeysAreOrderedAndDisjoint(t *testing.T) {
+ const begin = int64(1_700_000_000_000_000_000)
+ covers := TimestampCoverKeys(sortableInt64(begin), sortableInt64(begin+int64(6*time.Hour)))
+ previousEnd := []byte(nil)
+ for index, cover := range covers {
+ if len(cover.Start) == 0 {
+ t.Fatalf("cover %d has an empty start key", index)
+ }
+ if previousEnd != nil && bytes.Compare(cover.Start, previousEnd) < 0 {
+ t.Fatalf("cover %d starts at %x, before the previous run's end %x", index, cover.Start, previousEnd)
+ }
+ if cover.End != nil && bytes.Compare(cover.Start, cover.End) >= 0 {
+ t.Fatalf("cover %d is empty: start %x end %x", index, cover.Start, cover.End)
+ }
+ previousEnd = cover.End
+ }
+}
+
+func TestTimestampCoverKeysSpanTheFullDomain(t *testing.T) {
+ // A cover over the entire 64-bit domain must still produce usable bounds.
+ // It legitimately collapses to the top shift level: every lower level is
+ // wholly contained, so splitRange recurses into them instead of emitting.
+ covers := TimestampCoverKeys(sortableInt64(-1<<63), sortableInt64(1<<63-1))
+ if len(covers) == 0 {
+ t.Fatal("cover of the whole domain produced no runs")
+ }
+ for index, cover := range covers {
+ if len(cover.Start) == 0 {
+ t.Fatalf("cover %d has an empty start key", index)
+ }
+ if cover.End != nil && bytes.Compare(cover.Start, cover.End) >= 0 {
+ t.Fatalf("cover %d is empty: start %x end %x", index, cover.Start, cover.End)
+ }
+ }
+ t.Logf("whole-domain cover: %d run(s), first shift byte 0x%x", len(covers), covers[0].Start[0])
+}
+
+func TestMaxSortableBucketIsTheValueRange(t *testing.T) {
+ // maxSortableBucket must be the largest value of ts>>shift, and last+1 must
+ // still encode distinctly below it, which is what lets a run be bounded by
+ // its own successor's key rather than the next shift level.
+ for shift := uint(0); shift <= 60; shift += 4 {
+ limit := maxSortableBucket(shift)
+ if uint64(1)<<(64-shift)-1 != limit {
+ t.Errorf("shift %d: maxSortableBucket = %d, want %d", shift, limit, uint64(1)<<(64-shift)-1)
+ }
+ if limit == 0 {
+ t.Errorf("shift %d: maxSortableBucket is zero", shift)
+ continue
+ }
+ atLimit := encodePrefixCodedSortableShift(limit<<shift, shift)
+ aboveLimit := encodePrefixCodedSortableShift(limit<<shift, shift)
+ if !bytes.Equal(atLimit, aboveLimit) {
+ t.Errorf("shift %d: the top bucket is not stable", shift)
+ }
+ // One below the limit must still round-trip distinctly.
+ below := encodePrefixCodedSortableShift((limit-1)<<shift, shift)
+ if bytes.Compare(below, atLimit) >= 0 {
+ t.Errorf("shift %d: bucket %d does not sort below the top bucket", shift, limit-1)
+ }
+ // The bound a run ending at the limit uses must be usable, except at
+ // the top shift level where nothing sorts above it and nil -- which an
+ // iterator reads as unbounded -- is the only correct end.
+ end := coverRunEnd(shift, limit)
+ if shift < 60 && end == nil {
+ t.Errorf("shift %d: a run at the top bucket has no end key", shift)
+ }
+ if shift == 60 && end != nil {
+ t.Errorf("shift 60: end key = %x, want nil for the top level", end)
+ }
+ }
+}
diff --git a/pkg/index/internal/nativeice/numeric_range_test.go b/pkg/index/internal/nativeice/numeric_range_test.go
new file mode 100644
index 000000000..e119814a0
--- /dev/null
+++ b/pkg/index/internal/nativeice/numeric_range_test.go
@@ -0,0 +1,268 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nativeice
+
+import (
+ "bytes"
+ "math"
+ "math/rand"
+ "testing"
+)
+
+type sortableRun struct {
+ shift uint
+ first, last uint64
+}
+
+func coverOf(lo, hi uint64) []sortableRun {
+ runs := make([]sortableRun, 0, 64)
+ SplitSortableRange(lo, hi, func(shift uint, first, last uint64) {
+ runs = append(runs, sortableRun{shift: shift, first: first, last: last})
+ })
+ return runs
+}
+
+// covered reports whether a sortable value falls in the cover.
+func covered(runs []sortableRun, value uint64) bool {
+ for _, run := range runs {
+ shifted := value >> run.shift
+ if shifted >= run.first && shifted <= run.last {
+ return true
+ }
+ }
+ return false
+}
+
+// checkCover asserts the two properties the query path relies on: the cover is
+// exact in both directions, and it stays within the documented term bound.
+func checkCover(t *testing.T, lo, hi uint64) {
+ t.Helper()
+ runs := coverOf(lo, hi)
+ if len(runs) == 0 {
+ t.Fatalf("cover of [%d, %d] is empty", lo, hi)
+ }
+ if len(runs) > sortableRangeMaxTerms {
+ t.Fatalf("cover of [%d, %d] used %d terms, above the %d bound", lo, hi, len(runs), sortableRangeMaxTerms)
+ }
+ for _, run := range runs {
+ if run.first > run.last {
+ t.Fatalf("cover of [%d, %d] emitted an inverted run %+v", lo, hi, run)
+ }
+ if run.shift > 60 {
+ t.Fatalf("cover of [%d, %d] emitted shift %d, above the writer's top level", lo, hi, run.shift)
+ }
+ }
+ // Endpoints and their immediate neighbors pin down both directions
+ // without walking a 64-bit span.
+ probes := []uint64{lo, hi}
+ if lo > 0 {
+ probes = append(probes, lo-1)
+ }
+ if hi < math.MaxUint64 {
+ probes = append(probes, hi+1)
+ }
+ // Values sharing a bucket with each endpoint must be covered when the
+ // endpoint is, which is what makes the cover exact for a whole bucket.
+ for _, probe := range []uint64{lo, hi} {
+ for shift := uint(0); shift <= 60; shift += 4 {
+ probes = append(probes, probe&^(uint64(0xF)<<shift), probe|(uint64(0xF)<<shift))
+ }
+ }
+ for _, probe := range probes {
+ want := probe >= lo && probe <= hi
+ if got := covered(runs, probe); got != want {
+ t.Fatalf("cover of [%d, %d]: value %d covered=%v, want %v (runs %+v)", lo, hi, probe, got, want, runs)
+ }
+ }
+}
+
+func TestSplitSortableRangeCoversExactly(t *testing.T) {
+ cases := [][2]uint64{
+ {0, 0},
+ {0, 15},
+ {0, 16},
+ {0, 17},
+ {15, 15},
+ {16, 31},
+ {1, 14},
+ {5, 10},
+ {1 << 32, 1<<32 + 15},
+ {0, math.MaxUint64},
+ {math.MaxUint64 - 1, math.MaxUint64},
+ {0x8000000000000000, 0x8000000000000000},
+ {0x7FFFFFFFFFFFFFFF, 0x8000000000000000},
+ {0, 0x8000000000000000},
+ }
+ for _, testCase := range cases {
+ checkCover(t, testCase[0], testCase[1])
+ }
+}
+
+func TestSplitSortableRangeMatchesBruteForce(t *testing.T) {
+ // Fixed seed: a failure has to be reproducible from the printed case.
+ source := rand.New(rand.NewSource(20261009)) //nolint:gosec // deterministic test fixture, not security
+ for iteration := 0; iteration < 10000; iteration++ {
+ var lo, hi uint64
+ switch iteration % 4 {
+ case 0:
+ lo, hi = source.Uint64(), source.Uint64()
+ case 1:
+ // Small ranges, which exercise the fine levels.
+ lo = source.Uint64() % 4096
+ hi = lo + source.Uint64()%4096
+ case 2:
+ // Ranges aligned to a level, which skip whole levels.
+ shift := uint(source.Intn(16)) * 4
+ lo = source.Uint64() &^ (uint64(0xF) << shift)
+ hi = lo | (uint64(0xF) << shift)
+ default:
+ // Ranges spanning the sign flip in sortable space.
+ lo = 0x8000000000000000 - source.Uint64()%1024
+ hi = 0x8000000000000000 + source.Uint64()%1024
+ if lo > hi {
+ lo, hi = hi, lo
+ }
+ }
+ if lo > hi {
+ lo, hi = hi, lo
+ }
+ runs := coverOf(lo, hi)
+ // Exhaustively verify narrow ranges so an unsound bucket cannot hide
+ // in a region the probe set missed.
+ if hi-lo <= 8192 {
+ for value := lo; ; value++ {
+ if !covered(runs, value) {
+ t.Fatalf("cover of [%d, %d] misses %d", lo, hi, value)
+ }
+ if value == hi {
+ break
+ }
+ }
+ if lo > 0 && covered(runs, lo-1) {
+ t.Fatalf("cover of [%d, %d] includes %d below the range", lo, hi, lo-1)
+ }
+ if hi < math.MaxUint64 && covered(runs, hi+1) {
+ t.Fatalf("cover of [%d, %d] includes %d above the range", lo, hi, hi+1)
+ }
+ }
+ checkCover(t, lo, hi)
+ }
+}
+
+func TestSplitSortableRangeTermBoundIsTight(t *testing.T) {
+ // The widest cover a 64-bit range produces must stay within the bound the
+ // query path budgets dictionary lookups against.
+ worst := 0
+ source := rand.New(rand.NewSource(7)) //nolint:gosec // deterministic test fixture, not security
+ for iteration := 0; iteration < 2000; iteration++ {
+ lo := source.Uint64() & 0x0FFF0FFF0FFF0FFF
+ hi := source.Uint64() & 0x0FFF0FFF0FFF0FFF
+ if lo > hi {
+ lo, hi = hi, lo
+ }
+ if terms := len(coverOf(lo, hi)); terms > worst {
+ worst = terms
+ }
+ }
+ if worst > sortableRangeMaxTerms {
+ t.Fatalf("widest cover was %d terms, above the %d bound", worst, sortableRangeMaxTerms)
+ }
+ t.Logf("widest observed cover: %d terms (bound %d)", worst, sortableRangeMaxTerms)
+}
+
+func TestSplitSortableRangeRealisticWindowIsSmall(t *testing.T) {
+ // A 15-minute window expressed in nanoseconds, at a plausible epoch: the
+ // case the design motivates, and one that must not cost hundreds of
+ // dictionary lookups per segment.
+ const begin = int64(1_700_000_000_000_000_000)
+ const fifteenMinutes = int64(15 * 60 * 1e9)
+ terms := len(coverOf(sortableInt64(begin), sortableInt64(begin+fifteenMinutes-1)))
+ if terms > 64 {
+ t.Fatalf("a 15-minute window needed %d terms, want a few dozen", terms)
+ }
+ t.Logf("15-minute window: %d terms", terms)
+}
+
+func TestEncodePrefixCodedSortableShiftMatchesTheSignedEncoder(t *testing.T) {
+ values := []int64{0, 1, -1, 42, -42, math.MaxInt64, math.MinInt64, math.MinInt64 + 1}
+ source := rand.New(rand.NewSource(11)) //nolint:gosec // deterministic test fixture, not security
+ for iteration := 0; iteration < 2000; iteration++ {
+ values = append(values, int64(source.Uint64()))
+ }
+ for _, value := range values {
+ for shift := uint(0); shift <= 60; shift += 4 {
+ want := EncodePrefixCodedInt64Shift(value, shift)
+ got := encodePrefixCodedSortableShift(sortableInt64(value), shift)
+ if !bytes.Equal(want, got) {
+ t.Fatalf("value %d shift %d: sortable encoder = %x, signed encoder = %x", value, shift, got, want)
+ }
+ }
+ }
+}
+
+func TestEncodePrefixCodedSortableShiftPreservesOrder(t *testing.T) {
+ // Encoded terms at a fixed shift must be non-decreasing in sortable order,
+ // which is what lets a cover be turned into a dictionary range. Two values
+ // sharing a bucket at that shift encode identically, so strictness is only
+ // required when they differ at that shift.
+ values := []int64{math.MinInt64, -1_000_000, -1, 0, 1, 1_000_000, math.MaxInt64}
+ for shift := uint(0); shift <= 60; shift += 4 {
+ for outer := 0; outer+1 < len(values); outer++ {
+ left, right := values[outer], values[outer+1]
+ leftEncoded := encodePrefixCodedSortableShift(sortableInt64(left), shift)
+ rightEncoded := encodePrefixCodedSortableShift(sortableInt64(right), shift)
+ if string(leftEncoded) > string(rightEncoded) {
+ t.Fatalf("shift %d: %d encoded to %x, above %d's %x",
+ shift, left, leftEncoded, right, rightEncoded)
+ }
+ sameBucket := sortableInt64(left)>>shift == sortableInt64(right)>>shift
+ if !sameBucket && bytes.Equal(leftEncoded, rightEncoded) {
+ t.Fatalf("shift %d: %d and %d are in different buckets but both encoded to %x",
+ shift, left, right, leftEncoded)
+ }
+ }
+ }
+}
+
+func TestSplitSortableRangeEmitsNonOverlappingBuckets(t *testing.T) {
+ // The emitted runs must partition the range: no value may sit in two runs,
+ // or the cover would double-count documents.
+ source := rand.New(rand.NewSource(23)) //nolint:gosec // deterministic test fixture, not security
+ for iteration := 0; iteration < 5000; iteration++ {
+ lo := source.Uint64() % (1 << uint(source.Intn(48)+1))
+ hi := lo + source.Uint64()%(1<<uint(source.Intn(20)))
+ if hi-lo > 4096 {
+ hi = lo + 4096
+ }
+ runs := coverOf(lo, hi)
+ hits := make([]int, len(runs))
+ for value := lo; ; value++ {
+ for index, run := range runs {
+ if (value>>run.shift) >= run.first && (value>>run.shift) <= run.last {
+ hits[index]++
+ }
+ }
+ if value == hi {
+ break
+ }
+ }
+ for index, run := range runs {
+ if hits[index] == 0 {
+ t.Fatalf("cover of [%d, %d] emitted an empty run %+v at index %d", lo, hi, run, index)
+ }
+ }
+ }
+}
diff --git a/pkg/index/metrics/metrics.go b/pkg/index/metrics/metrics.go
index dc1a70680..e9209b70d 100644
--- a/pkg/index/metrics/metrics.go
+++ b/pkg/index/metrics/metrics.go
@@ -58,6 +58,12 @@ type Metrics struct {
cacheEntriesCount meter.Gauge
cacheBytesSize meter.Gauge
cacheMaxBytesSize meter.Gauge
+
+ // nativeTimeSegments and nativeTimeCandidatesPruned back NativeTimeMetrics,
+ // the native time-range pruning observability surface (native-index-time-
+ // pruning design, §6).
+ nativeTimeSegments meter.Counter
+ nativeTimeCandidatesPruned meter.Counter
}
// NewMetrics creates a new Metrics for a native-owner-backed index.
@@ -93,6 +99,9 @@ func NewMetrics(factory observability.Factory, labelNames ...string) *Metrics {
cacheEntriesCount: factory.NewGauge("inverted_index_cache_entries_count", labelNames...),
cacheBytesSize: factory.NewGauge("inverted_index_cache_bytes_size", labelNames...),
cacheMaxBytesSize: factory.NewGauge("inverted_index_cache_max_bytes_size", labelNames...),
+
+ nativeTimeSegments: factory.NewCounter("native_time_segments_total", append(labelNames, "class")...),
+ nativeTimeCandidatesPruned: factory.NewCounter("native_time_candidates_pruned_total", labelNames...),
}
}
@@ -134,6 +143,57 @@ func (m *Metrics) DeleteAll(labelValues ...string) {
m.cacheEntriesCount.Delete(labelValues...)
m.cacheBytesSize.Delete(labelValues...)
m.cacheMaxBytesSize.Delete(labelValues...)
+
+ for _, class := range nativeTimeClasses {
+ m.nativeTimeSegments.Delete(append(append([]string{}, labelValues...), class)...)
+ }
+ m.nativeTimeCandidatesPruned.Delete(labelValues...)
+}
+
+// nativeTimeClasses are every value the "class" label on
+// native_time_segments_total takes, mirrored here so DeleteAll can clear each
+// one. pkg/index/native classifies a segment into exactly one of these per
+// query (native-index-time-pruning design, §4.2, §4.5).
+var nativeTimeClasses = [...]string{"disjoint", "contained", "overlap_trie", "overlap_fallback"}
+
+// NativeTimeMetrics returns a recorder bound to labelValues -- the same
+// per-owner identity every other metric on m uses -- implementing the
+// pkg/index/native.TimeMetrics capability. This package does not import
+// native: the two methods below satisfy that interface structurally.
+func (m *Metrics) NativeTimeMetrics(labelValues ...string) *NativeTimeMetrics {
+ if m == nil {
+ return nil
+ }
+ return &NativeTimeMetrics{metrics: m, labelValues: append([]string(nil), labelValues...)}
+}
+
+// NativeTimeMetrics records native time-range pruning counts for one bound
+// label set. A nil *NativeTimeMetrics is valid everywhere its methods are
+// called and drops every count, so native.OwnerOptions.TimeMetrics may be set
+// unconditionally from NativeTimeMetrics's result even when the underlying
+// *Metrics is nil.
+type NativeTimeMetrics struct {
+ metrics *Metrics
+ labelValues []string
+}
+
+// IncTimeSegments records one segment's time-range classification: one of
+// nativeTimeClasses. It implements pkg/index/native.TimeMetrics.
+func (r *NativeTimeMetrics) IncTimeSegments(class string) {
+ if r == nil || r.metrics == nil {
+ return
+ }
+ r.metrics.nativeTimeSegments.Inc(1, append(append([]string(nil), r.labelValues...), class)...)
+}
+
+// AddTimeCandidatesPruned adds delta candidates removed by intersecting a
+// segment's candidates with the _timestamp trie's coverage. It implements
+// pkg/index/native.TimeMetrics.
+func (r *NativeTimeMetrics) AddTimeCandidatesPruned(delta uint64) {
+ if r == nil || r.metrics == nil || delta == 0 {
+ return
+ }
+ r.metrics.nativeTimeCandidatesPruned.Inc(float64(delta), r.labelValues...)
}
// ObserveNative records metrics available from the native owner: the live
diff --git a/pkg/index/native/merge_staging.go b/pkg/index/native/merge_staging.go
index 335e060cd..3d5908b30 100644
--- a/pkg/index/native/merge_staging.go
+++ b/pkg/index/native/merge_staging.go
@@ -198,6 +198,10 @@ func newSegmentFromMergeFile(path string, stats nativeice.MergeStats) (rootSegme
handle := &segmentHandle{
reader: reader, sourcePath: path, staged: true,
count: stats.DocumentCount, size: stats.Size, indexedFields: fields,
+ // The merge folds these from the surviving timestamped documents, so
+ // unlike a flush segment a merged one always knows its bounds unless no
+ // document carried a timestamp.
+ timeMin: stats.TimeMin, timeMax: stats.TimeMax, hasTime: stats.HasTime,
}
handle.refs.Store(1)
// Best effort, see newMemorySegment. A merged segment is exactly the case
diff --git a/pkg/index/native/owner.go b/pkg/index/native/owner.go
index c08a9e082..4334d21a4 100644
--- a/pkg/index/native/owner.go
+++ b/pkg/index/native/owner.go
@@ -224,6 +224,10 @@ type OwnerOptions struct {
// previous release's writer produces so a rolled-back node can still read
// document identity back from a segment this owner wrote.
IdentifierDocValues bool
+ // TimeMetrics receives time-range pruning classification and savings
+ // counts from every query this owner serves. Nil, the default, drops
+ // every count.
+ TimeMetrics TimeMetrics
}
// Owner serializes mutation admission and publishes immutable roots. A root
@@ -376,6 +380,14 @@ type segmentHandle struct {
fieldNamesOnce sync.Once
fieldNameSet map[string]struct{}
fieldNameErr error
+
+ // trieOnce caches whether this segment carries the coarse _timestamp term
+ // levels a time-range cover relies on. A segment without them would
+ // silently lose documents under a trie intersection, so it falls back to
+ // the per-document time check instead.
+ trieOnce sync.Once
+ trieUsableValue bool
+ trieUsableErr error
}
type persistedPromotion struct {
@@ -2220,9 +2232,16 @@ func newSegmentFromPayload(payload []byte, segmentID uint64) (rootSegment, error
}
// The reader borrows payload, which the caller hands over; the handle
// keeps it alive for the reader's lifetime.
+ // A merged segment records its time bounds in the footer, so read them here
+ // too. Without this the in-memory merge path leaves a merged segment
+ // reporting "no timestamps", which is what made segment-level time pruning
+ // useless for exactly the segments that hold most of the data.
+ timeMin, timeMax := reader.TimeBounds()
handle := &segmentHandle{
reader: reader, payload: payload, count: reader.DocumentCount(),
id: segmentID, size: uint64(len(payload)), persisted: atomic.Bool{},
+ timeMin: uint64(timeMin), timeMax: uint64(timeMax),
+ hasTime: timeMin != 0 || timeMax != 0,
}
handle.refs.Store(1)
if fields, fieldsErr := reader.Fields(); fieldsErr == nil {
@@ -2390,6 +2409,37 @@ func (s *memorySegment) MatchTerms(ctx context.Context, request MatchRequest) (M
series[number] = struct{}{}
}
}
+ // Narrow the term posting to the query's time range before visiting any
+ // document, so a segment the range misses costs no decode and a segment it
+ // partly covers does not decode the documents it excludes.
+ class := classifyTime(s.handle, request.TimeRange)
+ if class == timeDisjoint {
+ recordTimeClass(request.timeMetrics, "disjoint")
+ return MatchResult{}, nil
+ }
+ if class == timeContained {
+ recordTimeClass(request.timeMetrics, "contained")
+ }
+ timeExact := false
+ if class == timeOverlap {
+ beforeCardinality := postingCardinality(posting)
+ // Below trieMinCandidates, decoding the posting's few candidates costs
+ // less than OR-ing the cover's postings, exactly as narrowCandidatesToRange
+ // decides for every other entry point (§4.6): a point lookup with a tiny
+ // posting must not pay to build a cover spanning a wide range.
+ if s.handle.trieUsable() && beforeCardinality >= trieMinCandidates {
+ inRange, narrowErr := trieCandidates(ctx, s, request.TimeRange)
+ if narrowErr != nil {
+ return MatchResult{}, narrowErr
+ }
+ posting = narrowTermPostingToBitmap(posting, inRange)
+ timeExact = true
+ recordTimeClass(request.timeMetrics, "overlap_trie")
+ recordTimeCandidatesPruned(request.timeMetrics, beforeCardinality-postingCardinality(posting))
+ } else {
+ recordTimeClass(request.timeMetrics, "overlap_fallback")
+ }
+ }
result := MatchResult{}
for _, documentNumber := range postingDocuments(posting) {
if err := ctx.Err(); err != nil {
@@ -2434,8 +2484,13 @@ func (s *memorySegment) MatchTerms(ctx context.Context, request MatchRequest) (M
if identifier == nil {
return MatchResult{}, fmt.Errorf("document %d has no identifier: %w", documentNumber, ErrInvalidDocument)
}
- if request.TimeRange != nil && (!hasTimestamp || !request.TimeRange.contains(timestamp)) {
- continue
+ if request.TimeRange != nil {
+ if !hasTimestamp {
+ continue
+ }
+ if !timeExact && !request.TimeRange.contains(timestamp) {
+ continue
+ }
}
result.Identifiers = append(result.Identifiers, identifier)
result.Timestamps = append(result.Timestamps, timestamp)
@@ -2524,6 +2579,32 @@ func (d *disposals) wait() {
}
}
+// narrowTermPostingToBitmap restricts a term posting to the documents in
+// bitmap.
+//
+// It narrows Document numbers because that is what postingDocuments, the only
+// reader of a posting here, iterates: the Bitmap field exists for engines that
+// can combine postings directly, but this path does not. Filtering in place
+// keeps the allocation free.
+func narrowTermPostingToBitmap(posting nativeice.TermPosting, bitmap *roaringpkg.Bitmap) nativeice.TermPosting {
+ if posting.OneHit {
+ if !bitmap.Contains(uint32(posting.DocumentNumber)) {
+ posting.OneHit = false
+ posting.DocumentNumber = 0
+ posting.Documents = nil
+ }
+ return posting
+ }
+ kept := posting.Documents[:0]
+ for _, number := range posting.Documents {
+ if bitmap.Contains(uint32(number)) {
+ kept = append(kept, number)
+ }
+ }
+ posting.Documents = kept
+ return posting
+}
+
func postingDocuments(posting nativeice.TermPosting) []uint64 {
if posting.OneHit {
return []uint64{posting.DocumentNumber}
@@ -2531,6 +2612,15 @@ func postingDocuments(posting nativeice.TermPosting) []uint64 {
return posting.Documents
}
+// postingCardinality returns how many documents posting carries, without
+// allocating: a OneHit posting always carries exactly one.
+func postingCardinality(posting nativeice.TermPosting) uint64 {
+ if posting.OneHit {
+ return 1
+ }
+ return uint64(len(posting.Documents))
+}
+
func finishCallback(callback func(error), err error) error {
if callback != nil {
callback(err)
@@ -2598,6 +2688,11 @@ type MatchRequest struct {
SeriesField string
SeriesID []byte
TimeRange *TimeRange
+ // timeMetrics receives this request's time-range pruning outcome. It is
+ // unexported: only (*ReadView).MatchTerms sets it, from the owner it was
+ // acquired against, so a caller outside this package cannot set it and
+ // always gets the zero value (no recording).
+ timeMetrics TimeMetrics
}
// TimeRange is an inclusive/exclusive timestamp interval.
@@ -2622,6 +2717,9 @@ func (v *ReadView) MatchTerms(ctx context.Context, request MatchRequest) (MatchR
if err := v.check(ctx); err != nil {
return MatchResult{}, err
}
+ if v.owner != nil {
+ request.timeMetrics = v.owner.options.TimeMetrics
+ }
result := MatchResult{}
for _, current := range v.root.segments {
segmentResult, err := current.MatchTerms(ctx, request)
diff --git a/pkg/index/native/query.go b/pkg/index/native/query.go
index 514751368..bd9dc8719 100644
--- a/pkg/index/native/query.go
+++ b/pkg/index/native/query.go
@@ -172,11 +172,21 @@ func (v *ReadView) MatchTermsSet(ctx context.Context, request TermSetRequest) ([
if err != nil {
return nil, err
}
- totalCandidates += candidates.GetCardinality()
+ narrowed, projection, skip, err := narrowSegmentForTime(ctx, v.owner, segment, candidates, request.Scope.TimeRange)
+ if err != nil {
+ return nil, err
+ }
+ if skip {
+ continue
+ }
+ // Counted after narrowing: a document the range excludes was never a
+ // candidate the query has to consider, so it should not spend the
+ // caller's limit.
+ totalCandidates += narrowed.GetCardinality()
if request.MaxCandidates != 0 && totalCandidates > request.MaxCandidates {
return nil, ErrQueryLimit
}
- hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, candidates, request.Scope.TimeRange)
+ hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, narrowed, request.Scope.TimeRange, projection)
if err != nil {
return nil, err
}
@@ -253,7 +263,14 @@ func (v *ReadView) MatchAllTermSets(ctx context.Context, requests []TermSetReque
break
}
}
- hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, candidates, requests[0].Scope.TimeRange)
+ narrowed, projection, skip, narrowErr := narrowSegmentForTime(ctx, v.owner, segment, candidates, requests[0].Scope.TimeRange)
+ if narrowErr != nil {
+ return nil, narrowErr
+ }
+ if skip {
+ continue
+ }
+ hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, narrowed, requests[0].Scope.TimeRange, projection)
if err != nil {
return nil, err
}
@@ -369,7 +386,14 @@ func (v *ReadView) MatchField(ctx context.Context, request FieldRequest) ([]Quer
if request.MaxCandidates != 0 && totalCandidates > request.MaxCandidates {
return nil, ErrQueryLimit
}
- hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, candidates, request.Scope.TimeRange)
+ narrowed, projection, skip, err := narrowSegmentForTime(ctx, v.owner, segment, candidates, request.Scope.TimeRange)
+ if err != nil {
+ return nil, err
+ }
+ if skip {
+ continue
+ }
+ hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, narrowed, request.Scope.TimeRange, projection)
if err != nil {
return nil, err
}
@@ -402,11 +426,18 @@ func (v *ReadView) MatchRange(ctx context.Context, request RangeRequest) ([]Quer
if err != nil {
return nil, err
}
- totalCandidates += candidates.GetCardinality()
+ narrowed, projection, skip, err := narrowSegmentForTime(ctx, v.owner, segment, candidates, request.Scope.TimeRange)
+ if err != nil {
+ return nil, err
+ }
+ if skip {
+ continue
+ }
+ totalCandidates += narrowed.GetCardinality()
if request.MaxCandidates != 0 && totalCandidates > request.MaxCandidates {
return nil, ErrQueryLimit
}
- hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, candidates, request.Scope.TimeRange)
+ hits, err := projectCandidates(ctx, uint64(segmentIndex), segment, narrowed, request.Scope.TimeRange, projection)
if err != nil {
return nil, err
}
@@ -918,7 +949,64 @@ func seriesCandidates(ctx context.Context, segment *memorySegment, scope QuerySc
return result, nil
}
-func projectCandidates(ctx context.Context, segmentIndex uint64, segment *memorySegment, candidates *roaringpkg.Bitmap, timeRange *TimeRange) ([]QueryHit, error) {
+// timeProjection says what projection must do with a candidate's timestamp.
+type timeProjection uint8
+
+const (
+ // timeProjectionCheck compares each decoded timestamp against the range,
+ // which is what a segment the range only partly covers, or one that cannot
+ // use the trie, still needs.
+ timeProjectionCheck timeProjection = iota
+ // timeProjectionContained skips the range comparison because every
+ // timestamped document in the segment is already known to be in range,
+ // either because the range contains the segment or because the trie
+ // selected exactly the in-range documents. Documents without a timestamp
+ // are still dropped, which the hasTimestamp check keeps doing.
+ timeProjectionContained
+)
+
+// narrowSegmentForTime classifies the segment against the query range and
+// returns the candidates projection should walk. It records the
+// classification and, when the trie narrows an existing candidate set, the
+// number of candidates it removed, on owner's TimeMetrics.
+//
+// skip is true when the segment cannot contribute any hit, so the caller can
+// move on without decoding anything and without counting those candidates
+// against its limit. The returned candidate set is already narrowed, so a
+// limit check after this point no longer counts documents the range excludes.
+func narrowSegmentForTime(
+ ctx context.Context,
+ owner *Owner,
+ segment *memorySegment,
+ candidates *roaringpkg.Bitmap,
+ timeRange *TimeRange,
+) (narrowed *roaringpkg.Bitmap, projection timeProjection, skip bool, err error) {
+ class := classifyTime(segment.handle, timeRange)
+ if class == timeDisjoint {
+ owner.recordTimeClass("disjoint")
+ return nil, timeProjectionContained, true, nil
+ }
+ before := cardinalityOf(segment, candidates)
+ narrowed, exact, narrowErr := narrowCandidatesToRange(ctx, segment, candidates, class, timeRange)
+ if narrowErr != nil {
+ return nil, timeProjectionContained, false, narrowErr
+ }
+ projection = timeProjectionCheck
+ if exact {
+ projection = timeProjectionContained
+ }
+ recordSegmentTimeClass(owner, class, exact, before, narrowed.GetCardinality())
+ return narrowed, projection, false, nil
+}
+
+func projectCandidates(
+ ctx context.Context,
+ segmentIndex uint64,
+ segment *memorySegment,
+ candidates *roaringpkg.Bitmap,
+ timeRange *TimeRange,
+ projection timeProjection,
+) ([]QueryHit, error) {
if candidates == nil || candidates.IsEmpty() {
return nil, nil
}
@@ -962,8 +1050,18 @@ func projectCandidates(ctx context.Context, segmentIndex uint64, segment *memory
if identifier == nil {
return nil, fmt.Errorf("document %d has no identifier: %w", documentNumber, ErrCorrupt)
}
- if timeRange != nil && (!hasTimestamp || !timeRange.contains(timestamp)) {
- continue
+ // A time range is the only thing that requires a document to carry a
+ // timestamp: with no range, an untimestamped document is a normal
+ // result. Where there is a range, the document must have one, and the
+ // range comparison itself is only needed when the candidate set is not
+ // already known to be in range.
+ if timeRange != nil {
+ if !hasTimestamp {
+ continue
+ }
+ if projection == timeProjectionCheck && !timeRange.contains(timestamp) {
+ continue
+ }
}
result = append(result, QueryHit{Identifier: identifier, SeriesID: seriesID, Timestamp: timestamp, Segment: segmentIndex, DocumentNumber: documentNumber})
}
diff --git a/pkg/index/native/query_test.go b/pkg/index/native/query_test.go
index dab00b368..6d0b6da88 100644
--- a/pkg/index/native/query_test.go
+++ b/pkg/index/native/query_test.go
@@ -171,6 +171,49 @@ func TestNativeSortCursorKeysetPagesAndMissingValues(t *testing.T) {
require.Equal(t, []string{"d0", "d2", "d5", "d1"}, sortedIDs(page))
}
+// TestNativeSortCursorMaxCandidatesIsCumulativeAcrossSegments guards
+// NewSortCursor's documented contract ("A positive MaxCandidates is enforced
+// globally across segments"): two segments each contribute a candidate count
+// under the limit, but their sum is over it.
+func TestNativeSortCursorMaxCandidatesIsCumulativeAcrossSegments(t *testing.T) {
+ owner, err := NewOwner(OwnerOptions{Lease: testLease{}})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, owner.Close()) }()
+ // Two Batch calls land in two segments (seedTimePruneOwner-style corpora
+ // merge these; a single Acquire here does not), each holding two documents
+ // the status selection matches.
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: []Document{
+ {Identifier: []byte("a0"), Timestamp: 10, Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true},
+ {Name: "sort", Value: []byte("a"), Sort: true},
+ }},
+ {Identifier: []byte("a1"), Timestamp: 20, Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true},
+ {Name: "sort", Value: []byte("b"), Sort: true},
+ }},
+ }}))
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: []Document{
+ {Identifier: []byte("b0"), Timestamp: 30, Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true},
+ {Name: "sort", Value: []byte("c"), Sort: true},
+ }},
+ {Identifier: []byte("b1"), Timestamp: 40, Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true},
+ {Name: "sort", Value: []byte("d"), Sort: true},
+ }},
+ }}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ defer func() { require.NoError(t, view.Close()) }()
+ require.Len(t, view.root.segments, 2, "the fixture needs two segments for a cross-segment limit to mean anything")
+
+ _, err = view.NewSortCursor(context.Background(), SortCursorRequest{
+ SortField: "sort", PageSize: 10,
+ Selection: TermSetRequest{Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm, MaxCandidates: 3},
+ })
+ require.ErrorIs(t, err, ErrQueryLimit, "2 candidates in segment 1 plus 2 in segment 2 must exceed a limit of 3")
+}
+
func TestNativeSortCursorPinnedViewAndCancellation(t *testing.T) {
owner, err := NewOwner(OwnerOptions{Lease: testLease{}})
require.NoError(t, err)
diff --git a/pkg/index/native/sort_cursor.go b/pkg/index/native/sort_cursor.go
index aa6c8fa7a..08627b6ab 100644
--- a/pkg/index/native/sort_cursor.go
+++ b/pkg/index/native/sort_cursor.go
@@ -73,6 +73,12 @@ type sortCursorSegment struct {
segmentID uint64
candidates *roaringpkg.Bitmap
all bool
+ // timeExact records that every candidate here is already known to be in
+ // the query's time range, because the range contains the segment's bounds
+ // or the _timestamp trie selected exactly the in-range documents. It lets
+ // project skip the per-document range comparison while still requiring a
+ // timestamp.
+ timeExact bool
}
// SortCursor retains per-segment candidate bitmaps and one page-sized frontier
@@ -93,6 +99,47 @@ type SortCursor struct {
closed bool
}
+// sortCursorCandidates computes one segment's starting candidate set for a
+// sort cursor request, before time-range narrowing: a Range selection, an
+// empty selection (every ordinal, or a series' postings when scoped), or an
+// exact term-set selection. all reports the "every ordinal" representation
+// the empty, unscoped selection uses instead of a bitmap.
+func sortCursorCandidates(
+ ctx context.Context, segment *memorySegment, request SortCursorRequest, wildcards []nativeice.DictionaryAutomaton,
+) (candidates *roaringpkg.Bitmap, all bool, err error) {
+ switch {
+ case request.Range != nil:
+ if len(request.Selection.Terms) != 0 || request.Selection.Field != "" {
+ return nil, false, fmt.Errorf("sort cursor cannot combine range and exact selection: %w", ErrInvalidQuery)
+ }
+ rangeRequest := *request.Range
+ if rangeRequest.Scope.SeriesField == "" {
+ rangeRequest.Scope = request.Selection.Scope
+ }
+ candidates, err = rangeCandidates(ctx, segment, rangeRequest)
+ return candidates, false, err
+ case len(request.Selection.Terms) == 0 && len(request.Selection.Prefix) == 0 && len(request.Selection.Wildcard) == 0:
+ if request.Selection.Field != "" {
+ return nil, false, fmt.Errorf("sort cursor empty selection requires no field: %w", ErrQueryLimit)
+ }
+ if request.Selection.Scope.SeriesField == "" {
+ return nil, true, nil
+ }
+ candidates, err = seriesCandidates(ctx, segment, request.Selection.Scope)
+ return candidates, false, err
+ default:
+ if validateErr := validateTermSetRequest(request.Selection); validateErr != nil {
+ return nil, false, validateErr
+ }
+ // MaxCandidates is deferred to the cross-segment total check NewSortCursor
+ // makes after time narrowing, like the original per-term selection already did.
+ selection := request.Selection
+ selection.MaxCandidates = 0
+ candidates, err = exactCandidates(ctx, segment, selection, wildcards)
+ return candidates, false, err
+ }
+}
+
// NewSortCursor creates a keyset-paginated ordered cursor over one pinned
// root. A positive MaxCandidates is enforced globally across segments.
//
@@ -124,50 +171,50 @@ func (v *ReadView) NewSortCursor(ctx context.Context, request SortCursorRequest)
if !ok {
return nil, fmt.Errorf("sort cursor segment %d has type %T", index, current)
}
- var candidates *roaringpkg.Bitmap
- var candidateErr error
- all := false
- switch {
- case request.Range != nil:
- if len(request.Selection.Terms) != 0 || request.Selection.Field != "" {
- return nil, fmt.Errorf("sort cursor cannot combine range and exact selection: %w", ErrInvalidQuery)
- }
- rangeRequest := *request.Range
- if rangeRequest.Scope.SeriesField == "" {
- rangeRequest.Scope = request.Selection.Scope
- }
- candidates, candidateErr = rangeCandidates(ctx, segment, rangeRequest)
- if candidateErr != nil {
- return nil, candidateErr
- }
- total += candidates.GetCardinality()
- case len(request.Selection.Terms) == 0 && len(request.Selection.Prefix) == 0 && len(request.Selection.Wildcard) == 0:
- if request.Selection.Field != "" {
- return nil, fmt.Errorf("sort cursor empty selection requires no field: %w", ErrQueryLimit)
+ candidates, all, err := sortCursorCandidates(ctx, segment, request, selectionWildcards)
+ if err != nil {
+ return nil, err
+ }
+ // Narrow to the query's time range before the cursor retains anything.
+ // A disjoint segment is dropped outright; an overlapping one keeps only
+ // the documents the range can match, so the per-document comparison
+ // later is redundant for them.
+ // Same source the cursor projects against, so the narrowing and the
+ // later per-document check can never disagree.
+ timeRange := request.Selection.Scope.TimeRange
+ class := classifyTime(segment.handle, timeRange)
+ if class == timeDisjoint {
+ v.owner.recordTimeClass("disjoint")
+ continue
+ }
+ beforeCardinality := cardinalityOf(segment, candidates)
+ timeExact := true
+ if class == timeOverlap {
+ narrowed, exact, narrowErr := narrowCandidatesToRange(ctx, segment, candidates, class, timeRange)
+ if narrowErr != nil {
+ return nil, narrowErr
}
- if request.Selection.Scope.SeriesField == "" {
- all = true
- total += segment.handle.count
- } else {
- candidates, candidateErr = seriesCandidates(ctx, segment, request.Selection.Scope)
- if candidateErr != nil {
- return nil, candidateErr
+ if exact {
+ candidates, timeExact = narrowed, true
+ // The "all documents" path walks every ordinal. Once the trie
+ // has selected the in-range ones, the candidates are that set.
+ if all {
+ all = false
}
- total += candidates.GetCardinality()
- }
- default:
- if err := validateTermSetRequest(request.Selection); err != nil {
- return nil, err
- }
- // MaxCandidates is deferred to the cross-segment total check below,
- // like the original per-term selection already did.
- selection := request.Selection
- selection.MaxCandidates = 0
- var err error
- candidates, err = exactCandidates(ctx, segment, selection, selectionWildcards)
- if err != nil {
- return nil, err
+ } else {
+ timeExact = false
}
+ }
+ recordSegmentTimeClass(v.owner, class, timeExact, beforeCardinality, cardinalityOf(segment, candidates))
+ // Accumulated after narrowing, and after the "all documents" path has
+ // become an explicit candidate set, so a document the range excludes is
+ // never charged against the caller's limit. This must add to total, not
+ // replace it: MaxCandidates is a cross-segment budget (see NewSortCursor's
+ // doc comment), so a document decoded in an earlier segment this call
+ // still counts here.
+ if all {
+ total += segment.handle.count
+ } else if candidates != nil {
total += candidates.GetCardinality()
}
if request.Selection.MaxCandidates != 0 && total > request.Selection.MaxCandidates {
@@ -177,7 +224,7 @@ func (v *ReadView) NewSortCursor(ctx context.Context, request SortCursorRequest)
return nil, ErrQueryLimit
}
cursor.segments = append(cursor.segments, sortCursorSegment{
- segment: segment, segmentID: uint64(index), candidates: candidates, all: all,
+ segment: segment, segmentID: uint64(index), candidates: candidates, all: all, timeExact: timeExact,
})
}
return cursor, nil
@@ -221,7 +268,7 @@ func (c *SortCursor) NextPage(ctx context.Context) ([]SortedHit, error) {
return nil, err
}
documentNumber = nextDocument()
- hit, value, missing, err := c.project(current.segmentID, current.segment, documentNumber)
+ hit, value, missing, err := c.project(current.segmentID, current.segment, documentNumber, current.timeExact)
if err != nil {
if errors.Is(err, errSkipSortCandidate) {
continue
@@ -269,7 +316,7 @@ func (c *SortCursor) Close() error {
return nil
}
-func (c *SortCursor) project(segmentID uint64, segment *memorySegment, documentNumber uint64) (QueryHit, []byte, bool, error) {
+func (c *SortCursor) project(segmentID uint64, segment *memorySegment, documentNumber uint64, timeExact bool) (QueryHit, []byte, bool, error) {
if _, deleted := segment.deleted[documentNumber]; deleted {
return QueryHit{}, nil, true, errSkipSortCandidate
}
@@ -306,8 +353,13 @@ func (c *SortCursor) project(segmentID uint64, segment *memorySegment, documentN
if identifier == nil {
return QueryHit{}, nil, false, fmt.Errorf("document %d has no identifier: %w", documentNumber, ErrCorrupt)
}
- if c.timeRange != nil && (!hasTimestamp || !c.timeRange.contains(timestamp)) {
- return QueryHit{}, nil, true, errSkipSortCandidate
+ if c.timeRange != nil {
+ if !hasTimestamp {
+ return QueryHit{}, nil, true, errSkipSortCandidate
+ }
+ if !timeExact && !c.timeRange.contains(timestamp) {
+ return QueryHit{}, nil, true, errSkipSortCandidate
+ }
}
values, valuesErr := segment.handle.reader.DocumentValues(c.sortField, documentNumber)
if valuesErr != nil {
diff --git a/pkg/index/native/time_prune.go b/pkg/index/native/time_prune.go
new file mode 100644
index 000000000..69ae22562
--- /dev/null
+++ b/pkg/index/native/time_prune.go
@@ -0,0 +1,425 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package native
+
+import (
+ "context"
+ "fmt"
+ "math"
+
+ roaringpkg "github.com/RoaringBitmap/roaring"
+
+ "github.com/apache/skywalking-banyandb/pkg/index/internal/nativeice"
+)
+
+// trieMinCandidates is the smallest candidate set worth narrowing with the
+// _timestamp trie. Below it, decoding the few candidates costs less than
+// OR-ing the cover's postings, so the per-document time check stays. The
+// "all documents" path has no such candidate set and always builds the trie.
+//
+// Set from BenchmarkPointLookupCrossover (time_prune_bench_test.go), which
+// sweeps a point lookup's posting size against a whole-segment range: at 64
+// candidates the per-document check still wins (~280us CPU against the trie's
+// ~290us), and at 128 the trie wins decisively (~260us against ~450us), so
+// 128 is the smallest candidate count the trie reliably pays for.
+const trieMinCandidates = 128
+
+// TimeMetrics receives the outcome of a native query's time-range pruning
+// decisions (native-index-time-pruning design, §6). Implementations must be
+// safe for concurrent use: a query records inline, under no owner-wide lock.
+// A nil TimeMetrics is valid everywhere one is read and simply drops every
+// count, so OwnerOptions.TimeMetrics is optional.
+type TimeMetrics interface {
+ // IncTimeSegments records one segment's time-range classification:
+ // "disjoint", "contained", "overlap_trie", or "overlap_fallback".
+ IncTimeSegments(class string)
+ // AddTimeCandidatesPruned adds delta candidates removed from a segment's
+ // candidate set by intersecting it with the _timestamp trie's coverage.
+ AddTimeCandidatesPruned(delta uint64)
+}
+
+// recordTimeClass increments class's counter on tm, if tm is non-nil. class is
+// one of "disjoint", "contained", "overlap_trie", or "overlap_fallback".
+func recordTimeClass(tm TimeMetrics, class string) {
+ if tm == nil {
+ return
+ }
+ tm.IncTimeSegments(class)
+}
+
+// recordTimeCandidatesPruned adds delta to tm's pruned-candidates counter, if
+// tm is non-nil and delta is non-zero.
+func recordTimeCandidatesPruned(tm TimeMetrics, delta uint64) {
+ if tm == nil || delta == 0 {
+ return
+ }
+ tm.AddTimeCandidatesPruned(delta)
+}
+
+// recordTimeClass increments class's counter on the owner's TimeMetrics, if
+// one was configured. class is one of "disjoint", "contained",
+// "overlap_trie", or "overlap_fallback".
+func (o *Owner) recordTimeClass(class string) {
+ if o == nil {
+ return
+ }
+ recordTimeClass(o.options.TimeMetrics, class)
+}
+
+// recordTimeCandidatesPruned adds delta to the owner's pruned-candidates
+// counter, if a TimeMetrics was configured.
+func (o *Owner) recordTimeCandidatesPruned(delta uint64) {
+ if o == nil {
+ return
+ }
+ recordTimeCandidatesPruned(o.options.TimeMetrics, delta)
+}
+
+// cardinalityOf returns candidates's size, or segment's full document count
+// when candidates is nil -- the "every document in the segment" the "all
+// documents" selection path represents without allocating a bitmap for it.
+func cardinalityOf(segment *memorySegment, candidates *roaringpkg.Bitmap) uint64 {
+ if candidates != nil {
+ return candidates.GetCardinality()
+ }
+ return segment.handle.count
+}
+
+// recordSegmentTimeClass records one segment's time-range classification on
+// owner's TimeMetrics, and -- when the trie narrowed an existing candidate
+// set -- how many candidates it removed. class must not be timeDisjoint: a
+// disjoint segment is recorded by its caller, which never reaches the
+// candidate narrowing this records the result of.
+func recordSegmentTimeClass(owner *Owner, class timeClass, timeExact bool, before, after uint64) {
+ switch {
+ case class == timeContained:
+ owner.recordTimeClass("contained")
+ case timeExact:
+ owner.recordTimeClass("overlap_trie")
+ owner.recordTimeCandidatesPruned(before - after)
+ default:
+ owner.recordTimeClass("overlap_fallback")
+ }
+}
+
+// timeClass is what a query's time range means for one segment.
+type timeClass uint8
+
+const (
+ // timeDisjoint: no document in the segment can match, so the segment is
+ // skipped whole.
+ timeDisjoint timeClass = iota
+ // timeContained: every timestamped document matches, so the per-document
+ // range comparison is redundant.
+ timeContained
+ // timeOverlap: the segment must be narrowed, either with the trie or with
+ // the per-document check.
+ timeOverlap
+)
+
+// stepInt64 returns value one step away from the range it bounds, saturating at
+// the int64 limits rather than wrapping.
+//
+// An exclusive lower bound has to move up by one and an exclusive upper bound
+// down by one, so the direction is named rather than implied.
+func stepInt64(value int64, increase bool) int64 {
+ if increase {
+ if value == -1<<63 {
+ return value
+ }
+ return value + 1
+ }
+ if value == 1<<63-1 {
+ return value
+ }
+ if value == -1<<63 {
+ return value
+ }
+ return value - 1
+}
+
+// emptyTimeRange reports whether a range admits no int64 at all, which has to
+// be decided before the bounds are stepped. Saturating a step at the limits
+// widens the range rather than narrowing it, so an exclusive lower of MaxInt64,
+// an exclusive upper of MinInt64, or a single value excluded at both ends are
+// all empty even though the saturated interval would look non-empty.
+func emptyTimeRange(timeRange *TimeRange) bool {
+ if timeRange.IncludesLower && timeRange.IncludesUpper {
+ return timeRange.Lower > timeRange.Upper
+ }
+ if !timeRange.IncludesLower && timeRange.Lower == math.MaxInt64 {
+ return true
+ }
+ if !timeRange.IncludesUpper && timeRange.Upper == math.MinInt64 {
+ return true
+ }
+ if !timeRange.IncludesLower && !timeRange.IncludesUpper && timeRange.Lower == timeRange.Upper {
+ return true
+ }
+ return timeRange.Lower > timeRange.Upper
+}
+
+// classifyTime reports what a query range means for a segment's documents.
+//
+// Unknown bounds classify as overlap, never as disjoint. That is the safety
+// property everything else rests on: a segment whose bounds are missing, or
+// whose bounds folded across a sign change so int64(min) > int64(max), keeps
+// exactly today's behavior instead of being pruned on a guess.
+func classifyTime(handle *segmentHandle, timeRange *TimeRange) timeClass {
+ if timeRange == nil {
+ // No restriction at all, so there is nothing to compare and nothing to
+ // narrow: the segment is fully covered.
+ return timeContained
+ }
+ if !handle.hasTime {
+ return timeOverlap
+ }
+ if emptyTimeRange(timeRange) {
+ // Nothing matches, whatever the segment holds.
+ return timeDisjoint
+ }
+ minBound := int64(handle.timeMin)
+ maxBound := int64(handle.timeMax)
+ if minBound > maxBound {
+ // The bounds were folded with an unsigned comparison across a sign
+ // change, which the merge does deliberately to stay byte-identical with
+ // the flush path. They still describe the right set, but they cannot be
+ // ordered, so they are treated as unknown rather than trusted.
+ return timeOverlap
+ }
+ lower := timeRange.Lower
+ if !timeRange.IncludesLower {
+ lower = stepInt64(lower, true)
+ }
+ upper := timeRange.Upper
+ if !timeRange.IncludesUpper {
+ upper = stepInt64(upper, false)
+ }
+ if lower > upper {
+ // An empty range matches nothing, whatever the segment holds.
+ return timeDisjoint
+ }
+ if maxBound < lower || minBound > upper {
+ return timeDisjoint
+ }
+ if lower <= minBound && maxBound <= upper {
+ return timeContained
+ }
+ return timeOverlap
+}
+
+// segmentHasTimestampField reports whether the segment carries a _timestamp
+// field at all. A segment without one has no documents a time range can match,
+// so the trie is trivially usable.
+func segmentHasTimestampField(handle *segmentHandle) (bool, error) {
+ fields, fieldsErr := handle.reader.Fields()
+ if fieldsErr != nil {
+ return false, fmt.Errorf("enumerate segment fields: %w", fieldsErr)
+ }
+ for _, field := range fields {
+ if field == timestampField {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+// segmentTrieUsable reports whether the segment's _timestamp dictionary carries
+// the coarse term levels the trie cover relies on.
+//
+// Every timestamped document is indexed at each precision level, so a segment
+// with timestamps that is missing either the shift-4 or the shift-60 level was
+// written without the trie, or with a different precision step. Trusting the
+// cover there would silently drop documents, so the segment falls back to the
+// per-document check instead.
+//
+//nolint:contextcheck // the nativeice iterator has no context-capable variant.
+func segmentTrieUsable(handle *segmentHandle) (bool, error) {
+ hasTimestampField, fieldsErr := segmentHasTimestampField(handle)
+ if fieldsErr != nil {
+ return false, fieldsErr
+ }
+ if !hasTimestampField {
+ return true, nil
+ }
+ // Probe both ends of the level range with single-key dictionary ranges:
+ // [0x24, 0x25) is shift 4 and [0x5C, 0x5D) is shift 60.
+ levels := [][2]byte{
+ {prefixCodedShiftStart + 4, prefixCodedShiftStart + 5},
+ {prefixCodedShiftStart + 60, prefixCodedShiftStart + 61},
+ }
+ for _, level := range levels {
+ iterator, iteratorErr := handle.reader.NewDictionaryTermIterator(timestampField, nil, []byte{level[0]}, []byte{level[1]})
+ if iteratorErr != nil {
+ return false, fmt.Errorf("probe _timestamp level 0x%x: %w", level[0], iteratorErr)
+ }
+ term, termErr := iterator.NextTerm()
+ closeErr := iterator.Close()
+ if termErr != nil {
+ return false, fmt.Errorf("read _timestamp level 0x%x: %w", level[0], termErr)
+ }
+ if closeErr != nil {
+ return false, fmt.Errorf("close _timestamp level 0x%x: %w", level[0], closeErr)
+ }
+ // NextTerm yields a nil term only once the range is exhausted, so a nil
+ // here means the level is absent and the cover cannot be trusted.
+ if term == nil {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+// prefixCodedShiftStart mirrors nativeice's shift header byte, which leads every
+// _timestamp term and therefore sorts all terms of one level together.
+const prefixCodedShiftStart = 0x20
+
+// trieUsable returns the segment's cached trie capability, computing it once.
+// The probe reads only the pinned reader and immutable handle fields, so it is
+// safe to run concurrently with queries on the same view.
+//
+// A segment that cannot be probed reports false rather than an error: it is
+// treated as unusable, which routes it to the per-document check. A wrong
+// answer here costs speed; the other direction would cost results.
+func (h *segmentHandle) trieUsable() bool {
+ h.trieOnce.Do(func() {
+ h.trieUsableValue, h.trieUsableErr = segmentTrieUsable(h)
+ })
+ return h.trieUsableErr == nil && h.trieUsableValue
+}
+
+// trieCandidates returns the exact set of documents in a segment whose
+// timestamp falls in the query range.
+//
+// It turns the range into a cover of _timestamp dictionary term ranges and ORs
+// their postings. Because the writer indexes every document under one term per
+// precision level, and the cover partitions the range into disjoint buckets, a
+// document is in the result exactly when its timestamp is in range.
+//
+// Documents without a timestamp carry no _timestamp terms and so are excluded,
+// which matches the contract that a time range never returns them. Deleted
+// documents may appear and are still filtered where they are today.
+//
+//nolint:contextcheck // the nativeice dictionary iterator has no context-capable variant.
+func trieCandidates(ctx context.Context, segment *memorySegment, timeRange *TimeRange) (*roaringpkg.Bitmap, error) {
+ result := roaringpkg.New()
+ lower := timeRange.Lower
+ if !timeRange.IncludesLower {
+ lower = stepInt64(lower, true)
+ }
+ upper := timeRange.Upper
+ if !timeRange.IncludesUpper {
+ upper = stepInt64(upper, false)
+ }
+ if lower > upper {
+ return result, nil
+ }
+ handle := segment.handle
+ covers := nativeice.TimestampCoverKeys(sortableInt64(lower), sortableInt64(upper))
+ for _, cover := range covers {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ iterator, iteratorErr := handle.reader.NewDictionaryTermIterator(timestampField, nil, cover.Start, cover.End)
+ if iteratorErr != nil {
+ return nil, fmt.Errorf("iterate _timestamp cover: %w", iteratorErr)
+ }
+ for {
+ if err := ctx.Err(); err != nil {
+ _ = iterator.Close()
+ return nil, err
+ }
+ term, termErr := iterator.NextTerm()
+ if termErr != nil {
+ _ = iterator.Close()
+ return nil, fmt.Errorf("read _timestamp cover term: %w", termErr)
+ }
+ if term == nil {
+ break
+ }
+ posting, found, postingErr := handle.reader.TermPostingBitmap(timestampField, term)
+ if postingErr != nil {
+ _ = iterator.Close()
+ return nil, fmt.Errorf("read _timestamp posting: %w", postingErr)
+ }
+ if found {
+ result.Or(posting)
+ }
+ }
+ if closeErr := iterator.Close(); closeErr != nil {
+ return nil, fmt.Errorf("close _timestamp cover iterator: %w", closeErr)
+ }
+ }
+ return result, nil
+}
+
+// sortableInt64 maps a signed int64 into unsigned order that matches signed
+// order, which is the transform every _timestamp term is built on.
+func sortableInt64(value int64) uint64 {
+ return uint64(value) ^ 0x8000000000000000
+}
+
+// narrowCandidatesToRange intersects a candidate set with the segment's
+// in-range documents, and reports whether the result is exact.
+//
+// exact is true when the caller may stop comparing each candidate's timestamp
+// against the range: either the segment is wholly contained, or the trie
+// already selected precisely the in-range documents. It is false when the
+// caller must keep the per-document check.
+//
+// candidates may be nil, meaning "every document in the segment"; the result is
+// then the trie set itself.
+func narrowCandidatesToRange(
+ ctx context.Context,
+ segment *memorySegment,
+ candidates *roaringpkg.Bitmap,
+ class timeClass,
+ timeRange *TimeRange,
+) (narrowed *roaringpkg.Bitmap, exact bool, err error) {
+ switch class {
+ case timeDisjoint:
+ return roaringpkg.New(), true, nil
+ case timeContained:
+ if candidates == nil {
+ // Every document, and every timestamped one is in range. Documents
+ // without a timestamp still have to be dropped, which the caller's
+ // hasTimestamp check does.
+ return nil, true, nil
+ }
+ return candidates, true, nil
+ default:
+ if !segment.handle.trieUsable() {
+ return candidates, false, nil
+ }
+ if candidates == nil {
+ trie, trieErr := trieCandidates(ctx, segment, timeRange)
+ if trieErr != nil {
+ return nil, false, trieErr
+ }
+ return trie, true, nil
+ }
+ // With few candidates the per-document check is cheaper than OR-ing the
+ // cover's postings.
+ if candidates.GetCardinality() < trieMinCandidates {
+ return candidates, false, nil
+ }
+ trie, trieErr := trieCandidates(ctx, segment, timeRange)
+ if trieErr != nil {
+ return nil, false, trieErr
+ }
+ return roaringpkg.And(candidates, trie), true, nil
+ }
+}
diff --git a/pkg/index/native/time_prune_bench_test.go b/pkg/index/native/time_prune_bench_test.go
new file mode 100644
index 000000000..a7227704e
--- /dev/null
+++ b/pkg/index/native/time_prune_bench_test.go
@@ -0,0 +1,441 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package native
+
+import (
+ "context"
+ "fmt"
+ "syscall"
+ "testing"
+
+ roaringpkg "github.com/RoaringBitmap/roaring"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/index/internal/nativeice"
+)
+
+// cpuNanos returns this process's accumulated user+system CPU time. Comparing
+// two readings around a benchmark loop reports process CPU independent of
+// scheduling noise, which is what the native-index-time-pruning design's
+// gates (§7.2) are measured against, not wall time.
+func cpuNanos() int64 {
+ var ru syscall.Rusage
+ _ = syscall.Getrusage(syscall.RUSAGE_SELF, &ru)
+ return ru.Utime.Nano() + ru.Stime.Nano()
+}
+
+// runBench runs fn b.N times, reporting allocations (B/op, allocs/op) and
+// process CPU per operation alongside testing.B's own wall-time ns/op.
+func runBench(b *testing.B, fn func()) {
+ b.Helper()
+ b.ReportAllocs()
+ c0 := cpuNanos()
+ b.ResetTimer()
+ for range b.N {
+ fn()
+ }
+ b.StopTimer()
+ b.ReportMetric(float64(cpuNanos()-c0)/float64(b.N), "cpu-ns/op")
+}
+
+const (
+ benchDocuments = 200_000
+ benchSpan = int64(24 * 60 * 60 * 1e9) // one day of nanoseconds
+ benchBase = int64(1_700_000_000_000_000_000)
+)
+
+// benchSegment builds a segment shaped like a stream element index over a day:
+// many documents spread evenly across the span, every one indexed on status
+// and carrying a timestamp.
+func benchSegment(b *testing.B) (*ReadView, *memorySegment) {
+ b.Helper()
+ owner := newTestOwnerB(b)
+ values := make([]Document, 0, benchDocuments)
+ for index := range benchDocuments {
+ values = append(values, Document{
+ Identifier: []byte(fmt.Sprintf("doc-%08d", index)),
+ Timestamp: benchBase + int64(index)*(benchSpan/int64(benchDocuments)),
+ Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true, Store: true},
+ {Name: "series", Value: []byte("series-a"), Index: true, Store: true},
+ },
+ })
+ }
+ require.NoError(b, owner.Batch(context.Background(), Batch{Documents: values}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(b, err)
+ b.Cleanup(func() { require.NoError(b, owner.Close()); require.NoError(b, view.Close()) })
+ segment, err := view.segmentAt(0)
+ require.NoError(b, err)
+ return view, segment
+}
+
+func newTestOwnerB(b *testing.B) *Owner {
+ b.Helper()
+ owner, err := NewOwner(OwnerOptions{Lease: testLease{}})
+ require.NoError(b, err)
+ return owner
+}
+
+// windowFraction names the slice of the segment's span a benchmark window covers.
+type windowFraction struct {
+ name string
+ lo, span int64
+}
+
+func benchWindows() []windowFraction {
+ return []windowFraction{
+ {"window=100%", 0, benchSpan},
+ {"window=10%", 0, benchSpan / 10},
+ {"window=1%", benchSpan / 100, benchSpan / 100},
+ {"window=0.1%", benchSpan / 1000, benchSpan / 1000},
+ }
+}
+
+// BenchmarkTimeRangeNarrowing compares the pruning path against the
+// per-document check it replaced, in one binary so the two are measured under
+// identical conditions.
+//
+// The baseline walks every candidate and decodes its stored fields before
+// comparing timestamps, which is exactly what every query did before. The
+// pruned path classifies the segment and intersects the candidates with the
+// trie first.
+func BenchmarkTimeRangeNarrowing(b *testing.B) {
+ for _, window := range benchWindows() {
+ b.Run(window.name, func(b *testing.B) {
+ _, segment := benchSegment(b)
+ lower := benchBase + window.lo
+ upper := lower + window.span
+ timeRange := &TimeRange{Lower: lower, Upper: upper, IncludesLower: true, IncludesUpper: true}
+ all := allOrdinals(segment.handle.count)
+
+ b.Run("perDocument", func(b *testing.B) {
+ runCountingBench(b, func() int { return baselineProject(b.Context(), segment, all, timeRange) })
+ })
+ b.Run("pruned", func(b *testing.B) {
+ runCountingBench(b, func() int {
+ narrowed, exact, err := narrowCandidatesToRange(
+ b.Context(), segment, all, timeOverlap, timeRange)
+ require.NoError(b, err)
+ require.True(b, exact)
+ return projectNarrowed(b.Context(), segment, narrowed, timeRange)
+ })
+ })
+ })
+ }
+}
+
+// baselineProject is the pre-change shape: decode every candidate's stored
+// fields, then drop the ones outside the range.
+func baselineProject(ctx context.Context, segment *memorySegment, candidates *roaringpkg.Bitmap, timeRange *TimeRange) int {
+ return visitAndCount(ctx, segment, candidates, timeRange, timeProjectionCheck)
+}
+
+func projectNarrowed(ctx context.Context, segment *memorySegment, candidates *roaringpkg.Bitmap, timeRange *TimeRange) int {
+ return visitAndCount(ctx, segment, candidates, timeRange, timeProjectionContained)
+}
+
+// visitAndCount walks candidates, decoding each one's stored _timestamp, and
+// counts the ones timeRange keeps -- every one of them when projection already
+// guarantees that, or only the in-range ones when it must still check.
+func visitAndCount(
+ ctx context.Context,
+ segment *memorySegment,
+ candidates *roaringpkg.Bitmap,
+ timeRange *TimeRange,
+ projection timeProjection,
+) int {
+ kept := 0
+ iterator := candidates.Iterator()
+ for iterator.HasNext() {
+ if err := ctx.Err(); err != nil {
+ return kept
+ }
+ documentNumber := uint64(iterator.Next())
+ var timestamp int64
+ var hasTimestamp bool
+ var err error
+ visitErr := segment.handle.reader.VisitDocument(documentNumber, func(document nativeice.StoredDocument) error {
+ return document.VisitStoredFields(func(name string, value []byte) bool {
+ if name == timestampField {
+ timestamp, err = nativeice.DecodePrefixCodedInt64(value)
+ hasTimestamp = err == nil
+ return err == nil
+ }
+ return true
+ })
+ })
+ if visitErr != nil || err != nil {
+ continue
+ }
+ if !hasTimestamp {
+ continue
+ }
+ if projection == timeProjectionCheck && !timeRange.contains(timestamp) {
+ continue
+ }
+ kept++
+ }
+ return kept
+}
+
+// runCountingBench is runBench for a benchmarked function that reports how
+// many documents it kept. Using the count (rather than discarding it) is what
+// makes the in-range comparison inside fn a real branch rather than dead
+// work a compiler could fold away, and the sanity check catches a window
+// fixture degenerate enough to keep nothing, which would silently turn a
+// timing benchmark into a no-op.
+func runCountingBench(b *testing.B, fn func() int) {
+ b.Helper()
+ var kept int
+ runBench(b, func() { kept = fn() })
+ require.NotZero(b, kept, "benchmark window must keep at least one document")
+}
+
+// BenchmarkSortCursorNarrowing measures the sort path, where an out-of-range
+// candidate additionally pays for a sort-value lookup and frontier work.
+func BenchmarkSortCursorNarrowing(b *testing.B) {
+ for _, window := range []windowFraction{{"window=1%", benchSpan / 100, benchSpan / 100}} {
+ b.Run(window.name, func(b *testing.B) {
+ _, segment := benchSegment(b)
+ lower := benchBase + window.lo
+ timeRange := &TimeRange{Lower: lower, Upper: lower + window.span, IncludesLower: true, IncludesUpper: true}
+ all := allOrdinals(segment.handle.count)
+
+ b.Run("perDocument", func(b *testing.B) {
+ runCountingBench(b, func() int { return baselineProject(b.Context(), segment, all, timeRange) })
+ })
+ b.Run("pruned", func(b *testing.B) {
+ runCountingBench(b, func() int {
+ narrowed, exact, err := narrowCandidatesToRange(
+ b.Context(), segment, all, timeOverlap, timeRange)
+ require.NoError(b, err)
+ require.True(b, exact)
+ return projectNarrowed(b.Context(), segment, narrowed, timeRange)
+ })
+ })
+ })
+ }
+}
+
+// BenchmarkTrieConstruction isolates the cost the pruning pays, which is what
+// the trieMinCandidates threshold exists to keep below the cost it saves.
+func BenchmarkTrieConstruction(b *testing.B) {
+ _, segment := benchSegment(b)
+ for _, window := range benchWindows() {
+ b.Run(window.name, func(b *testing.B) {
+ lower := benchBase + window.lo
+ timeRange := &TimeRange{Lower: lower, Upper: lower + window.span, IncludesLower: true, IncludesUpper: true}
+ runBench(b, func() {
+ if _, err := trieCandidates(b.Context(), segment, timeRange); err != nil {
+ b.Fatal(err)
+ }
+ })
+ })
+ }
+}
+
+// BenchmarkTimeRangeNarrowingAfterMerge repeats the 1% window comparison on a
+// segment produced by a real merge, which is where §4.1 fixed the time bounds
+// that pruning depends on: before that fix, every merged segment reported no
+// timestamps and classifyTime never pruned it.
+func BenchmarkTimeRangeNarrowingAfterMerge(b *testing.B) {
+ owner := newTestOwnerB(b)
+ for batch := range 4 {
+ values := make([]Document, 0, benchDocuments/4)
+ for index := range benchDocuments / 4 {
+ ordinal := batch*(benchDocuments/4) + index
+ values = append(values, Document{
+ Identifier: []byte(fmt.Sprintf("doc-%08d", ordinal)),
+ Timestamp: benchBase + int64(ordinal)*(benchSpan/int64(benchDocuments)),
+ Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true, Store: true},
+ {Name: "series", Value: []byte("series-a"), Index: true, Store: true},
+ },
+ })
+ }
+ require.NoError(b, owner.Batch(context.Background(), Batch{Documents: values}))
+ }
+ require.NoError(b, owner.forceMergeAll(context.Background()))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(b, err)
+ b.Cleanup(func() { require.NoError(b, owner.Close()); require.NoError(b, view.Close()) })
+ require.Len(b, view.root.segments, 1, "the fixture must merge down to one segment")
+ segment, err := view.segmentAt(0)
+ require.NoError(b, err)
+ require.True(b, segment.handle.hasTime, "a merged segment must carry time bounds (§4.1)")
+
+ window := windowFraction{"window=1%", benchSpan / 100, benchSpan / 100}
+ lower := benchBase + window.lo
+ upper := lower + window.span
+ timeRange := &TimeRange{Lower: lower, Upper: upper, IncludesLower: true, IncludesUpper: true}
+ all := allOrdinals(segment.handle.count)
+
+ b.Run("perDocument", func(b *testing.B) {
+ runCountingBench(b, func() int { return baselineProject(b.Context(), segment, all, timeRange) })
+ })
+ b.Run("pruned", func(b *testing.B) {
+ runCountingBench(b, func() int {
+ narrowed, exact, narrowErr := narrowCandidatesToRange(
+ b.Context(), segment, all, timeOverlap, timeRange)
+ require.NoError(b, narrowErr)
+ require.True(b, exact)
+ return projectNarrowed(b.Context(), segment, narrowed, timeRange)
+ })
+ })
+}
+
+// BenchmarkSortLimit20 measures an index-ordered LIMIT 20 query end to end,
+// through the real SortCursor, rather than the narrowCandidatesToRange proxy
+// BenchmarkSortCursorNarrowing uses. perDocument builds the same cursor shape
+// NewSortCursor used to always build -- "all" candidates, no time narrowing --
+// so the per-page walk still pays for every out-of-range sort-value decode;
+// pruned goes through NewSortCursor itself.
+func BenchmarkSortLimit20(b *testing.B) {
+ for _, window := range benchWindows() {
+ b.Run(window.name, func(b *testing.B) {
+ view, segment := benchSegment(b)
+ lower := benchBase + window.lo
+ upper := lower + window.span
+ timeRange := &TimeRange{Lower: lower, Upper: upper, IncludesLower: true, IncludesUpper: true}
+
+ b.Run("perDocument", func(b *testing.B) {
+ runBench(b, func() {
+ cursor := &SortCursor{
+ view: view, ctx: b.Context(), sortField: timestampField, pageSize: 20,
+ timeRange: timeRange,
+ segments: []sortCursorSegment{{segment: segment, segmentID: 0, all: true, timeExact: false}},
+ }
+ if _, pageErr := cursor.NextPage(b.Context()); pageErr != nil {
+ b.Fatal(pageErr)
+ }
+ })
+ })
+ b.Run("pruned", func(b *testing.B) {
+ runBench(b, func() {
+ cursor, cursorErr := view.NewSortCursor(b.Context(), SortCursorRequest{
+ SortField: timestampField, PageSize: 20,
+ Selection: TermSetRequest{Scope: QueryScope{TimeRange: timeRange}},
+ })
+ if cursorErr != nil {
+ b.Fatal(cursorErr)
+ }
+ if _, pageErr := cursor.NextPage(b.Context()); pageErr != nil {
+ b.Fatal(pageErr)
+ }
+ })
+ })
+ })
+ }
+}
+
+// pointLookupSegment builds a segment shaped like benchSegment, except exactly
+// hits documents (spread evenly across the span) additionally carry a "lookup"
+// field valued "hit", so MatchRequest{Field: "lookup", Term: []byte("hit")}
+// returns a posting of exactly that size.
+func pointLookupSegment(b *testing.B, documents, hits int) *memorySegment {
+ b.Helper()
+ owner := newTestOwnerB(b)
+ values := make([]Document, 0, documents)
+ stride := documents / hits
+ for index := range documents {
+ fields := []Field{
+ {Name: "status", Value: []byte("ok"), Index: true, Store: true},
+ {Name: "series", Value: []byte("series-a"), Index: true, Store: true},
+ }
+ if stride > 0 && index%stride == 0 {
+ fields = append(fields, Field{Name: "lookup", Value: []byte("hit"), Index: true, Store: true})
+ }
+ values = append(values, Document{
+ Identifier: []byte(fmt.Sprintf("doc-%08d", index)),
+ Timestamp: benchBase + int64(index)*(benchSpan/int64(documents)),
+ Fields: fields,
+ })
+ }
+ require.NoError(b, owner.Batch(context.Background(), Batch{Documents: values}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(b, err)
+ b.Cleanup(func() { require.NoError(b, owner.Close()); require.NoError(b, view.Close()) })
+ segment, err := view.segmentAt(0)
+ require.NoError(b, err)
+ return segment
+}
+
+// pointLookupPerDocument is MatchTerms's shape before the trieMinCandidates
+// threshold existed: decode every posting candidate's timestamp and compare.
+func pointLookupPerDocument(segment *memorySegment, posting nativeice.TermPosting, timeRange *TimeRange) int {
+ kept := 0
+ for _, number := range postingDocuments(posting) {
+ var timestamp int64
+ var hasTimestamp bool
+ _ = segment.handle.reader.VisitDocument(number, func(document nativeice.StoredDocument) error {
+ return document.VisitStoredFields(func(name string, value []byte) bool {
+ if name == timestampField {
+ ts, decodeErr := nativeice.DecodePrefixCodedInt64(value)
+ hasTimestamp = decodeErr == nil
+ if hasTimestamp {
+ timestamp = ts
+ }
+ return decodeErr == nil
+ }
+ return true
+ })
+ })
+ if hasTimestamp && timeRange.contains(timestamp) {
+ kept++
+ }
+ }
+ return kept
+}
+
+// pointLookupWithTrie is MatchTerms's shape once a posting clears
+// trieMinCandidates: narrow the posting against the trie's cover instead of
+// decoding each candidate.
+func pointLookupWithTrie(ctx context.Context, segment *memorySegment, posting nativeice.TermPosting, timeRange *TimeRange) (int, error) {
+ inRange, err := trieCandidates(ctx, segment, timeRange)
+ if err != nil {
+ return 0, err
+ }
+ return len(postingDocuments(narrowTermPostingToBitmap(posting, inRange))), nil
+}
+
+// BenchmarkPointLookupCrossover sweeps the size of the posting a point lookup
+// narrows, with the query range spanning the whole segment (the "a point
+// lookup by series with two hits in a long range" shape §4.6 names), to find
+// where building the trie's cover stops costing more than it saves.
+// trieMinCandidates (time_prune.go) is set from the crossover this reports.
+func BenchmarkPointLookupCrossover(b *testing.B) {
+ timeRange := &TimeRange{Lower: benchBase, Upper: benchBase + benchSpan, IncludesLower: true, IncludesUpper: true}
+ for _, hits := range []int{1, 8, 32, 64, 128, 256, 512, 1024, 4096} {
+ b.Run(fmt.Sprintf("postings=%d", hits), func(b *testing.B) {
+ segment := pointLookupSegment(b, benchDocuments, hits)
+ posting, found, err := segment.handle.reader.TermPosting("lookup", []byte("hit"))
+ require.NoError(b, err)
+ require.True(b, found)
+
+ b.Run("perDocument", func(b *testing.B) {
+ runBench(b, func() { pointLookupPerDocument(segment, posting, timeRange) })
+ })
+ b.Run("withTrie", func(b *testing.B) {
+ runBench(b, func() {
+ if _, trieErr := pointLookupWithTrie(b.Context(), segment, posting, timeRange); trieErr != nil {
+ b.Fatal(trieErr)
+ }
+ })
+ })
+ })
+ }
+}
diff --git a/pkg/index/native/time_prune_parity_test.go b/pkg/index/native/time_prune_parity_test.go
new file mode 100644
index 000000000..d80264937
--- /dev/null
+++ b/pkg/index/native/time_prune_parity_test.go
@@ -0,0 +1,562 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package native
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "sort"
+ "testing"
+
+ roaringpkg "github.com/RoaringBitmap/roaring"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/index/internal/nativeice"
+)
+
+// timePruneDocument builds a document with a timestamp and the fields the
+// parity tests select on.
+func timePruneDocument(identifier string, timestamp int64, status string) Document {
+ return Document{
+ Identifier: []byte(identifier), Timestamp: timestamp,
+ Fields: []Field{
+ {Name: "status", Value: []byte(status), Index: true, Store: true},
+ {Name: "series", Value: []byte("series-a"), Index: true, Store: true},
+ {Name: "value", Value: []byte(fmt.Sprintf("%d", timestamp)), Index: true, Store: true, Sort: true},
+ },
+ }
+}
+
+// timePruneWindows are the range shapes the design calls out: empty, point,
+// inclusive and exclusive edges, full span, and disjoint on both sides.
+func timePruneWindows() []*TimeRange {
+ const base = int64(1_700_000_000_000_000_000)
+ return []*TimeRange{
+ {Lower: base + 1000, Upper: base + 1000, IncludesLower: false, IncludesUpper: false},
+ {Lower: base + 1000, Upper: base + 1000, IncludesLower: true, IncludesUpper: true},
+ {Lower: base + 1000, Upper: base + 2000, IncludesLower: false, IncludesUpper: true},
+ {Lower: base + 1000, Upper: base + 2000, IncludesLower: true, IncludesUpper: false},
+ {Lower: base + 1000, Upper: base + 2000, IncludesLower: false, IncludesUpper: false},
+ {Lower: base, Upper: base + 10_000, IncludesLower: true, IncludesUpper: true},
+ {Lower: base - 10_000, Upper: base - 1000, IncludesLower: true, IncludesUpper: true},
+ {Lower: base + 100_000, Upper: base + 200_000, IncludesLower: true, IncludesUpper: true},
+ }
+}
+
+// identifiersOf sorts identifiers so two runs compare without depending on
+// segment or document order.
+func identifiersOf(hits []QueryHit) []string {
+ result := make([]string, 0, len(hits))
+ for _, hit := range hits {
+ result = append(result, string(hit.Identifier))
+ }
+ sort.Strings(result)
+ return result
+}
+
+func matchIdentifiersOf(result MatchResult) []string {
+ out := make([]string, 0, len(result.Identifiers))
+ for _, identifier := range result.Identifiers {
+ out = append(out, string(identifier))
+ }
+ sort.Strings(out)
+ return out
+}
+
+// seedTimePruneOwner writes a spread of documents, some deleted, so every
+// window shape has in-range, out-of-range and missing documents.
+func seedTimePruneOwner(t *testing.T, count int, seed int64) *Owner {
+ t.Helper()
+ owner := newTestOwner(t, nil)
+ const base = int64(1_700_000_000_000_000_000)
+ source := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic fixture
+ documents := make([]Document, 0, count)
+ for index := range count {
+ timestamp := base + int64(index)*100 + int64(source.Intn(100))
+ status := "ok"
+ if index%3 == 0 {
+ status = "bad"
+ }
+ documents = append(documents, timePruneDocument(fmt.Sprintf("doc-%04d", index), timestamp, status))
+ }
+ // Some documents carry no timestamp at all; a time range must never return
+ // them, but a query with no range must.
+ for index := 0; index < count/10; index++ {
+ documents = append(documents, Document{
+ Identifier: []byte(fmt.Sprintf("undated-%04d", index)),
+ Fields: []Field{
+ {Name: "status", Value: []byte("ok"), Index: true, Store: true},
+ {Name: "series", Value: []byte("series-a"), Index: true, Store: true},
+ },
+ })
+ }
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: documents}))
+ // Delete a slice of the middle so the range has to respect deletions.
+ deletes := make([][]byte, 0, count/10)
+ for index := count / 5; index < count/5+count/10; index++ {
+ deletes = append(deletes, []byte(fmt.Sprintf("doc-%04d", index)))
+ }
+ if len(deletes) > 0 {
+ require.NoError(t, owner.Batch(context.Background(), Batch{Deletes: deletes}))
+ }
+ return owner
+}
+
+// TestTimePruningMatchesThePerDocumentPath is the central correctness claim:
+// narrowing a segment by time must return exactly what checking each document's
+// timestamp returns, for every entry point, every window shape, and a segment
+// whose candidates are dense enough to take the trie path.
+func TestTimePruningMatchesThePerDocumentPath(t *testing.T) {
+ // Above trieMinCandidates, so the trie is actually exercised rather than
+ // the small-candidate fallback.
+ owner := seedTimePruneOwner(t, 2000, 3)
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+
+ windows := append(timePruneWindows(), nil)
+ for _, window := range windows {
+ name := "no-range"
+ if window != nil {
+ name = fmt.Sprintf("%d..%d[%v%v", window.Lower, window.Upper, window.IncludesLower, window.IncludesUpper)
+ }
+ t.Run(name, func(t *testing.T) {
+ scope := QueryScope{SeriesField: "series", SeriesID: []byte("series-a")}
+ scope.TimeRange = window
+
+ // MatchTermsSet: the exact-term filter path.
+ terms := TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm, Scope: scope,
+ }
+ narrowed, narrowedErr := view.MatchTermsSet(context.Background(), terms)
+ require.NoError(t, narrowedErr)
+
+ // The per-document reference: the same request with the range
+ // removed, filtered here by the rule the old code applied.
+ unranged := terms
+ unranged.Scope.TimeRange = nil
+ all, allErr := view.MatchTermsSet(context.Background(), unranged)
+ require.NoError(t, allErr)
+ require.Equal(t, referenceFiltered(all, window), identifiersOf(narrowed),
+ "MatchTermsSet disagreed with the per-document path")
+
+ // MatchField: the presence path, which relies on MatchField
+ // returning documents that never set the field.
+ fieldRequest := FieldRequest{Field: "series", Scope: scope, MaxTerms: 8}
+ narrowedField, fieldErr := view.MatchField(context.Background(), fieldRequest)
+ require.NoError(t, fieldErr)
+ unrangedField := fieldRequest
+ unrangedField.Scope.TimeRange = nil
+ allField, allFieldErr := view.MatchField(context.Background(), unrangedField)
+ require.NoError(t, allFieldErr)
+ require.Equal(t, referenceFiltered(allField, window), identifiersOf(narrowedField),
+ "MatchField disagreed with the per-document path")
+
+ // MatchTerms: the in-memory path.
+ matched, matchedErr := view.MatchTerms(context.Background(), MatchRequest{
+ Field: "status", Term: []byte("ok"), SeriesField: "series", SeriesID: []byte("series-a"),
+ TimeRange: window,
+ })
+ require.NoError(t, matchedErr)
+ unrangedMatched, unrangedMatchedErr := view.MatchTerms(context.Background(), MatchRequest{
+ Field: "status", Term: []byte("ok"), SeriesField: "series", SeriesID: []byte("series-a"),
+ })
+ require.NoError(t, unrangedMatchedErr)
+ require.Equal(t, referenceMatched(unrangedMatched, window), matchIdentifiersOf(matched),
+ "MatchTerms disagreed with the per-document path")
+
+ // MatchAllTermSets: the conjunction path. A single conjunct behaves
+ // like MatchTermsSet, which is enough to prove this entry point's own
+ // copy of the time-narrowing wiring agrees with the reference.
+ allSets, allSetsErr := view.MatchAllTermSets(context.Background(), []TermSetRequest{terms})
+ require.NoError(t, allSetsErr)
+ unrangedAllSets, unrangedAllSetsErr := view.MatchAllTermSets(context.Background(), []TermSetRequest{unranged})
+ require.NoError(t, unrangedAllSetsErr)
+ require.Equal(t, referenceFiltered(unrangedAllSets, window), identifiersOf(allSets),
+ "MatchAllTermSets disagreed with the per-document path")
+
+ // MatchRange: the encoded-byte-order path. ["a", "z"] covers every
+ // status value the fixture writes ("ok", "bad"), so it selects the
+ // same documents the unranged reference does.
+ rangeRequest := RangeRequest{
+ Field: "status", Lower: []byte("a"), Upper: []byte("z"),
+ IncludesLower: true, IncludesUpper: true, Scope: scope, MaxTerms: 8,
+ }
+ rangedHits, rangedErr := view.MatchRange(context.Background(), rangeRequest)
+ require.NoError(t, rangedErr)
+ unrangedRange := rangeRequest
+ unrangedRange.Scope.TimeRange = nil
+ unrangedRangeHits, unrangedRangeErr := view.MatchRange(context.Background(), unrangedRange)
+ require.NoError(t, unrangedRangeErr)
+ require.Equal(t, referenceFiltered(unrangedRangeHits, window), identifiersOf(rangedHits),
+ "MatchRange disagreed with the per-document path")
+ })
+ }
+}
+
+// referenceFiltered applies today's rule to an unranged result: a document with
+// no timestamp is dropped whenever a range is present, and an out-of-range
+// document is dropped.
+func referenceFiltered(hits []QueryHit, window *TimeRange) []string {
+ if window == nil {
+ return identifiersOf(hits)
+ }
+ kept := make([]string, 0, len(hits))
+ for _, hit := range hits {
+ if hit.Timestamp == 0 {
+ continue
+ }
+ if !window.contains(hit.Timestamp) {
+ continue
+ }
+ kept = append(kept, string(hit.Identifier))
+ }
+ sort.Strings(kept)
+ return kept
+}
+
+func referenceMatched(result MatchResult, window *TimeRange) []string {
+ if window == nil {
+ return matchIdentifiersOf(result)
+ }
+ kept := make([]string, 0, len(result.Identifiers))
+ for index, identifier := range result.Identifiers {
+ if result.Timestamps[index] == 0 {
+ continue
+ }
+ if !window.contains(result.Timestamps[index]) {
+ continue
+ }
+ kept = append(kept, string(identifier))
+ }
+ sort.Strings(kept)
+ return kept
+}
+
+// TestTimePruningFallsBackWithoutTheTrie checks that a natively written
+// segment reports a usable trie.
+func TestTimePruningFallsBackWithoutTheTrie(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ documents := make([]Document, 0, 600)
+ for index := range 600 {
+ documents = append(documents, timePruneDocument(fmt.Sprintf("doc-%04d", index), base+int64(index), "ok"))
+ }
+ owner := newTestOwner(t, nil)
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: documents}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+
+ segment, segmentErr := view.segmentAt(0)
+ require.NoError(t, segmentErr)
+ require.True(t, segment.handle.trieUsable(), "a natively written segment must report a usable trie")
+}
+
+// shiftZeroOnlySegment encodes a real segment through nativeice.EncodeSegment,
+// with a _timestamp field carrying only the shift-0 term -- no current writer
+// emits this, but the compatibility contract requires a segment missing the
+// coarse levels to fall back rather than silently losing documents. Unlike a
+// hand-built segmentHandle, this exercises the real segmentTrieUsable probe
+// against a real dictionary.
+//
+// The returned segment carries the single reference newSegmentFromPayload
+// creates it with -- the same reference a successful merge hands straight to
+// a published root without retaining again. The caller either appends it to
+// an owner's root (whose teardown releases it exactly once) or releases it
+// itself; it must not do both.
+func shiftZeroOnlySegment(t *testing.T, count int, base int64) *memorySegment {
+ t.Helper()
+ documents := make([]nativeice.EncodeDocument, 0, count)
+ var timeMin, timeMax int64
+ for index := range count {
+ timestamp := base + int64(index)
+ if index == 0 || timestamp < timeMin {
+ timeMin = timestamp
+ }
+ if index == 0 || timestamp > timeMax {
+ timeMax = timestamp
+ }
+ status := "ok"
+ if index%3 == 0 {
+ status = "bad"
+ }
+ documents = append(documents, nativeice.EncodeDocument{
+ Identifier: []byte(fmt.Sprintf("doc-%04d", index)),
+ Fields: []nativeice.EncodeField{
+ {
+ Name: "status", Value: []byte(status),
+ Terms: []nativeice.EncodeTerm{{Value: []byte(status), Frequency: 1}},
+ Index: true, Store: true,
+ },
+ {
+ Name: "series", Value: []byte("series-a"),
+ Terms: []nativeice.EncodeTerm{{Value: []byte("series-a"), Frequency: 1}},
+ Index: true, Store: true,
+ },
+ {
+ Name: timestampField, Value: nativeice.EncodePrefixCodedInt64(timestamp),
+ Terms: []nativeice.EncodeTerm{{Value: nativeice.EncodePrefixCodedInt64Shift(timestamp, 0), Frequency: 1}},
+ Index: true, Store: true, Sort: true,
+ },
+ },
+ })
+ }
+ payload, encodeErr := nativeice.EncodeSegment(nativeice.Generation{
+ Documents: documents, SegmentID: 1,
+ TimeMin: uint64(timeMin), TimeMax: uint64(timeMax),
+ })
+ require.NoError(t, encodeErr)
+ root, segmentErr := newSegmentFromPayload(payload, 1)
+ require.NoError(t, segmentErr)
+ segment, ok := root.(*memorySegment)
+ require.True(t, ok, "newSegmentFromPayload returned %T", root)
+ return segment
+}
+
+// TestTimePruningFallsBackForASegmentWithoutCoarseTerms builds a segment
+// through the real encoder emitting only shift-0 _timestamp terms, proves the
+// real probe rejects it, and proves the fallback still returns exactly what
+// the per-document path returns.
+func TestTimePruningFallsBackForASegmentWithoutCoarseTerms(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ // The owner is created first so a failed assertion below still leaves the
+ // segment reachable from a root whose teardown closes it; nothing retains
+ // it a second time, so Close releases it exactly once.
+ owner := newTestOwner(t, nil)
+ segment := shiftZeroOnlySegment(t, 600, base)
+ owner.root.segments = append(owner.root.segments, segment)
+
+ require.False(t, segment.handle.trieUsable(), "a segment missing the coarse _timestamp levels must not report a usable trie")
+
+ window := closedTestRange(base+100, base+200)
+ narrowed, exact, narrowErr := narrowCandidatesToRange(
+ context.Background(), segment, allOrdinals(segment.handle.count), timeOverlap, window)
+ require.NoError(t, narrowErr)
+ require.False(t, exact, "a segment that cannot use the trie must not claim an exact narrowing")
+ require.Equal(t, uint64(600), narrowed.GetCardinality(), "the fallback must leave the candidate set untouched")
+
+ // The fallback must still answer every entry point correctly end to end,
+ // not only at the narrowCandidatesToRange seam.
+ view, viewErr := owner.Acquire(context.Background())
+ require.NoError(t, viewErr)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+
+ for _, testWindow := range append(timePruneWindows(), window) {
+ ranged, rangedErr := view.MatchTermsSet(context.Background(), TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm,
+ Scope: QueryScope{TimeRange: testWindow},
+ })
+ require.NoError(t, rangedErr)
+ unranged, unrangedErr := view.MatchTermsSet(context.Background(), TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm,
+ })
+ require.NoError(t, unrangedErr)
+ require.Equal(t, referenceFiltered(unranged, testWindow), identifiersOf(ranged),
+ "fallback segment disagreed with the per-document path for %+v", testWindow)
+ }
+}
+
+func closedTestRange(lower, upper int64) *TimeRange {
+ return &TimeRange{Lower: lower, Upper: upper, IncludesLower: true, IncludesUpper: true}
+}
+
+func allOrdinals(count uint64) *roaringpkg.Bitmap {
+ bitmap := roaringpkg.New()
+ for value := uint64(0); value < count; value++ {
+ bitmap.Add(uint32(value))
+ }
+ return bitmap
+}
+
+// TestTrieIsActuallyUsed guards the optimisation: without this, a refactor
+// could silently fall back everywhere and every other test would still pass.
+func TestTrieIsActuallyUsed(t *testing.T) {
+ owner := seedTimePruneOwner(t, 2000, 5)
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+ segment, segmentErr := view.segmentAt(0)
+ require.NoError(t, segmentErr)
+
+ const base = int64(1_700_000_000_000_000_000)
+ window := closedTestRange(base+200, base+200_000)
+ narrowed, exact, narrowErr := narrowCandidatesToRange(
+ context.Background(), segment, allOrdinals(segment.handle.count), timeOverlap, window)
+ require.NoError(t, narrowErr)
+ require.True(t, exact, "a wide overlap on a usable segment must narrow exactly")
+ require.Less(t, narrowed.GetCardinality(), segment.handle.count,
+ "the trie must actually remove out-of-range documents")
+ require.NotZero(t, narrowed.GetCardinality(), "the trie must keep in-range documents")
+}
+
+// TestDisjointSegmentVisitsNothing proves the strongest saving: a segment the
+// range misses must not decode a single document. An empty result could also
+// happen if every candidate were decoded and then filtered out, so the test
+// hook counts the reader's VisitDocument calls directly rather than trusting
+// the result shape.
+func TestDisjointSegmentVisitsNothing(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ owner := newTestOwner(t, nil)
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: []Document{
+ timePruneDocument("a", base, "ok"), timePruneDocument("b", base+10, "ok"),
+ }}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+
+ segment, segmentErr := view.segmentAt(0)
+ require.NoError(t, segmentErr)
+ before := segment.handle.reader.VisitDocumentCalls()
+
+ far := closedTestRange(base+1_000_000, base+2_000_000)
+ hits, hitsErr := view.MatchTermsSet(context.Background(), TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm,
+ Scope: QueryScope{TimeRange: far},
+ })
+ require.NoError(t, hitsErr)
+ require.Empty(t, hits, "a disjoint segment must contribute nothing")
+ require.Equal(t, before, segment.handle.reader.VisitDocumentCalls(),
+ "a disjoint segment must not decode a single document")
+}
+
+// TestTimePruningSurvivesMergedSegments checks the pruning still works after a
+// merge, which is where the bounds used to be missing entirely.
+func TestTimePruningSurvivesMergedSegments(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ // The in-memory merge path writes the same footer bounds and terms as the
+ // on-disk staged one, which pkg/index/internal/nativeice already covers.
+ owner := newTestOwner(t, nil)
+ for batch := range 4 {
+ documents := make([]Document, 0, 500)
+ for index := range 500 {
+ ordinal := batch*500 + index
+ documents = append(documents, timePruneDocument(
+ fmt.Sprintf("doc-%04d", ordinal), base+int64(ordinal)*10, "ok"))
+ }
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: documents}))
+ }
+ before, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ require.Greater(t, len(before.root.segments), 1, "the fixture needs several segments to merge")
+ segmentCount := len(before.root.segments)
+ require.NoError(t, before.Close())
+
+ // The tiered planner never schedules below two comparably sized segments
+ // in a tier, which this small fixture may not satisfy; force the merge so
+ // the test is about merged segments rather than about the planner. The view
+ // is released first, because compacting under a pinned root is rejected.
+ require.NoError(t, owner.forceMergeAll(context.Background()))
+
+ merged, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, merged.Close()) })
+ require.Less(t, len(merged.root.segments), segmentCount, "compaction should have merged segments")
+
+ // A merged segment must now carry bounds, which is the prerequisite the
+ // whole design rests on.
+ for index, current := range merged.root.segments {
+ segment := current.(*memorySegment)
+ require.True(t, segment.handle.hasTime, "merged segment %d has no time bounds", index)
+ }
+
+ for _, window := range timePruneWindows() {
+ narrowed, narrowedErr := merged.MatchTermsSet(context.Background(), TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm,
+ Scope: QueryScope{TimeRange: window},
+ })
+ require.NoError(t, narrowedErr)
+ unranged, unrangedErr := merged.MatchTermsSet(context.Background(), TermSetRequest{
+ Field: "status", Terms: [][]byte{[]byte("ok")}, Mode: MatchAnyTerm,
+ })
+ require.NoError(t, unrangedErr)
+ require.Equal(t, referenceFiltered(unranged, window), identifiersOf(narrowed),
+ "merged segments disagreed with the per-document path for %+v", window)
+ }
+}
+
+// TestTrieCoverageMatchesThePerDocumentSet is the exactness claim in its purest
+// form: the trie's document set has to equal a brute-force scan of the segment.
+func TestTrieCoverageMatchesThePerDocumentSet(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ source := rand.New(rand.NewSource(99)) //nolint:gosec // deterministic fixture
+ documents := make([]Document, 0, 800)
+ timestamps := make([]int64, 0, 800)
+ for index := range 800 {
+ timestamp := base + int64(source.Intn(2_000_000))
+ timestamps = append(timestamps, timestamp)
+ documents = append(documents, timePruneDocument(fmt.Sprintf("doc-%04d", index), timestamp, "ok"))
+ }
+ owner := newTestOwner(t, nil)
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: documents}))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+ segment, segmentErr := view.segmentAt(0)
+ require.NoError(t, segmentErr)
+
+ for iteration := 0; iteration < 200; iteration++ {
+ lower := base + int64(source.Intn(2_000_000))
+ upper := lower + int64(source.Intn(50_000))
+ window := closedTestRange(lower, upper)
+ bitmap, bitmapErr := trieCandidates(context.Background(), segment, window)
+ require.NoError(t, bitmapErr)
+
+ expected := make(map[uint32]struct{})
+ for index, timestamp := range timestamps {
+ if timestamp >= lower && timestamp <= upper {
+ expected[uint32(index)] = struct{}{}
+ }
+ }
+ require.Equal(t, len(expected), int(bitmap.GetCardinality()),
+ "trie cardinality differs from brute force for [%d, %d]", lower, upper)
+ for ordinal := range expected {
+ require.True(t, bitmap.Contains(ordinal),
+ "trie is missing ordinal %d for [%d, %d]", ordinal, lower, upper)
+ }
+ }
+}
+
+// TestTimestampTermsSurviveAMerge keeps the trie honest after a merge, since a
+// merge rewrites every term.
+func TestTimestampTermsSurviveAMerge(t *testing.T) {
+ const base = int64(1_700_000_000_000_000_000)
+ owner := newTestOwner(t, nil)
+ for batch := range 3 {
+ documents := make([]Document, 0, 400)
+ for index := range 400 {
+ ordinal := batch*400 + index
+ documents = append(documents, timePruneDocument(
+ fmt.Sprintf("doc-%04d", ordinal), base+int64(ordinal)*37, "ok"))
+ }
+ require.NoError(t, owner.Batch(context.Background(), Batch{Documents: documents}))
+ }
+ // Compact before acquiring: a pinned root makes the merge stale.
+ require.NoError(t, owner.forceMergeAll(context.Background()))
+ view, err := owner.Acquire(context.Background())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, view.Close()) })
+
+ for index, current := range view.root.segments {
+ segment := current.(*memorySegment)
+ require.True(t, segment.handle.trieUsable(), "merged segment %d lost its coarse timestamp terms", index)
+
+ window := closedTestRange(base+1000, base+5000)
+ bitmap, bitmapErr := trieCandidates(context.Background(), segment, window)
+ require.NoError(t, bitmapErr)
+ require.NotZero(t, bitmap.GetCardinality(), "merged segment %d returned nothing for an in-range window", index)
+ }
+}
diff --git a/pkg/index/native/time_prune_test.go b/pkg/index/native/time_prune_test.go
new file mode 100644
index 000000000..a8caed73d
--- /dev/null
+++ b/pkg/index/native/time_prune_test.go
@@ -0,0 +1,271 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. Apache Software
+// Foundation (ASF) licenses this file to you under the Apache License, Version
+// 2.0 (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package native
+
+import (
+ "math"
+ "testing"
+)
+
+// handleWithBounds builds a handle whose footer slots hold the given
+// timestamps, taking int64 so the uint64(int64) conversion the owner performs
+// happens here rather than on a constant.
+func handleWithBounds(minBound, maxBound int64, hasTime bool) *segmentHandle {
+ return &segmentHandle{timeMin: uint64(minBound), timeMax: uint64(maxBound), hasTime: hasTime}
+}
+
+func closedRange(lower, upper int64) *TimeRange {
+ return &TimeRange{Lower: lower, Upper: upper, IncludesLower: true, IncludesUpper: true}
+}
+
+func TestClassifyTime(t *testing.T) {
+ tests := []struct {
+ handle *segmentHandle
+ timeRange *TimeRange
+ name string
+ want timeClass
+ }{
+ {
+ name: "no range is fully covered",
+ handle: handleWithBounds(100, 200, true),
+ timeRange: nil,
+ want: timeContained,
+ },
+ {
+ name: "unknown bounds never prune",
+ handle: handleWithBounds(0, 0, false),
+ timeRange: closedRange(100, 200),
+ want: timeOverlap,
+ },
+ {
+ name: "bounds inverted by a sign change are unknown",
+ // This is what an unsigned fold over a set spanning zero produces:
+ // uint64(-5000) is the larger slot, so the pair reads back inverted.
+ handle: handleWithBounds(5000, -5000, true),
+ timeRange: closedRange(-10, 10),
+ want: timeOverlap,
+ },
+ {
+ name: "segment below the range",
+ handle: handleWithBounds(10, 20, true),
+ timeRange: closedRange(100, 200),
+ want: timeDisjoint,
+ },
+ {
+ name: "segment above the range",
+ handle: handleWithBounds(300, 400, true),
+ timeRange: closedRange(100, 200),
+ want: timeDisjoint,
+ },
+ {
+ name: "segment entirely inside the range",
+ handle: handleWithBounds(120, 180, true),
+ timeRange: closedRange(100, 200),
+ want: timeContained,
+ },
+ {
+ name: "segment exactly equal to the range",
+ handle: handleWithBounds(100, 200, true),
+ timeRange: closedRange(100, 200),
+ want: timeContained,
+ },
+ {
+ name: "segment straddling the lower edge",
+ handle: handleWithBounds(50, 150, true),
+ timeRange: closedRange(100, 200),
+ want: timeOverlap,
+ },
+ {
+ name: "segment straddling the upper edge",
+ handle: handleWithBounds(150, 250, true),
+ timeRange: closedRange(100, 200),
+ want: timeOverlap,
+ },
+ {
+ name: "segment enclosing the range",
+ handle: handleWithBounds(50, 250, true),
+ timeRange: closedRange(100, 200),
+ want: timeOverlap,
+ },
+ {
+ name: "exclusive lower excludes a touching segment",
+ handle: handleWithBounds(50, 100, true),
+ timeRange: &TimeRange{Lower: 100, Upper: 200, IncludesLower: false, IncludesUpper: true},
+ want: timeDisjoint,
+ },
+ {
+ name: "inclusive lower keeps a touching segment",
+ handle: handleWithBounds(50, 100, true),
+ timeRange: closedRange(100, 200),
+ want: timeOverlap,
+ },
+ {
+ name: "exclusive upper excludes a touching segment",
+ handle: handleWithBounds(200, 250, true),
+ timeRange: &TimeRange{Lower: 100, Upper: 200, IncludesLower: true, IncludesUpper: false},
+ want: timeDisjoint,
+ },
+ {
+ name: "empty exclusive range at one value",
+ handle: handleWithBounds(100, 200, true),
+ timeRange: &TimeRange{Lower: 100, Upper: 100, IncludesLower: false, IncludesUpper: true},
+ want: timeDisjoint,
+ },
+ {
+ name: "single inclusive value",
+ handle: handleWithBounds(100, 200, true),
+ timeRange: closedRange(150, 150),
+ want: timeOverlap,
+ },
+ {
+ name: "empty range after exclusive steps",
+ handle: handleWithBounds(100, 200, true),
+ timeRange: &TimeRange{Lower: 200, Upper: 100, IncludesLower: false, IncludesUpper: false},
+ want: timeDisjoint,
+ },
+ {
+ name: "exclusive bounds at the int64 limits do not wrap",
+ handle: handleWithBounds(math.MinInt64, math.MaxInt64, true),
+ timeRange: &TimeRange{Lower: math.MinInt64, Upper: math.MaxInt64, IncludesLower: false, IncludesUpper: false},
+ want: timeContained,
+ },
+ {
+ name: "bounds at the int64 limits stay ordered",
+ handle: handleWithBounds(math.MinInt64, math.MaxInt64, true),
+ timeRange: closedRange(math.MinInt64, math.MaxInt64),
+ want: timeContained,
+ },
+ {
+ name: "empty range at the int64 limits",
+ handle: handleWithBounds(math.MinInt64, math.MaxInt64, true),
+ timeRange: &TimeRange{Lower: math.MinInt64, Upper: math.MinInt64, IncludesLower: false, IncludesUpper: false},
+ want: timeDisjoint,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ if got := classifyTime(test.handle, test.timeRange); got != test.want {
+ t.Errorf("classifyTime = %v, want %v", got, test.want)
+ }
+ })
+ }
+}
+
+func TestClassifyTimeAgreesWithPerDocumentCheck(t *testing.T) {
+ // The classification has to agree with what the per-document check would
+ // decide, for every combination of bounds and window. A disagreement is
+ // either a lost document or a wrong one.
+ bounds := [][2]int64{
+ {0, 0}, {100, 100}, {50, 150}, {99, 101}, {100, 200}, {50, 250}, {200, 300}, {300, 400},
+ }
+ windows := []*TimeRange{
+ nil,
+ closedRange(100, 200),
+ closedRange(150, 150),
+ closedRange(0, 1000),
+ closedRange(120, 180),
+ closedRange(110, 190),
+ closedRange(100, 199),
+ closedRange(101, 200),
+ {Lower: 100, Upper: 200, IncludesLower: false, IncludesUpper: false},
+ closedRange(250, 350),
+ }
+ for _, bound := range bounds {
+ // A segment's bounds cover the timestamps it holds; a document at the
+ // low or high bound is inside the range whenever the segment overlaps.
+ handle := handleWithBounds(bound[0], bound[1], true)
+ for _, window := range windows {
+ class := classifyTime(handle, window)
+ if class == timeDisjoint {
+ if window == nil {
+ t.Fatalf("bounds %v: a nil range classified disjoint", bound)
+ }
+ lower := window.Lower
+ if !window.IncludesLower {
+ lower++
+ }
+ upper := window.Upper
+ if !window.IncludesUpper {
+ upper--
+ }
+ if lower <= upper && !(bound[1] < lower || bound[0] > upper) {
+ t.Fatalf("bounds %v window %+v: disjoint but the segment overlaps", bound, window)
+ }
+ }
+ if class == timeContained && window != nil {
+ lower := window.Lower
+ if !window.IncludesLower {
+ lower++
+ }
+ upper := window.Upper
+ if !window.IncludesUpper {
+ upper--
+ }
+ if !(lower <= bound[0] && bound[1] <= upper) {
+ t.Fatalf("bounds %v window %+v: contained but not fully covered", bound, window)
+ }
+ }
+ }
+ }
+}
+
+func TestEmptyTimeRange(t *testing.T) {
+ tests := []struct {
+ timeRange *TimeRange
+ want bool
+ }{
+ {closedRange(1, 2), false},
+ {&TimeRange{Lower: 2, Upper: 1, IncludesLower: true, IncludesUpper: true}, true},
+ {&TimeRange{Lower: 5, Upper: 5, IncludesLower: true, IncludesUpper: true}, false},
+ {&TimeRange{Lower: 5, Upper: 5, IncludesLower: false, IncludesUpper: false}, true},
+ {&TimeRange{Lower: 5, Upper: 5, IncludesLower: false, IncludesUpper: true}, false},
+ {&TimeRange{Lower: math.MaxInt64, Upper: math.MaxInt64, IncludesLower: false, IncludesUpper: true}, true},
+ {&TimeRange{Lower: math.MinInt64, Upper: math.MinInt64, IncludesLower: true, IncludesUpper: false}, true},
+ {&TimeRange{Lower: math.MinInt64, Upper: math.MaxInt64, IncludesLower: false, IncludesUpper: false}, false},
+ }
+ for _, test := range tests {
+ if got := emptyTimeRange(test.timeRange); got != test.want {
+ t.Errorf("emptyTimeRange(%+v) = %v, want %v", test.timeRange, got, test.want)
+ }
+ }
+}
+
+func TestStepInt64DoesNotWrap(t *testing.T) {
+ if got := stepInt64(math.MinInt64, false); got != math.MinInt64 {
+ t.Errorf("stepInt64(MinInt64, decrease) = %d, want MinInt64 rather than a wrap", got)
+ }
+ if got := stepInt64(math.MinInt64, true); got != math.MinInt64 {
+ t.Errorf("stepInt64(MinInt64, increase) = %d, want MinInt64", got)
+ }
+ if got := stepInt64(math.MaxInt64, false); got != math.MaxInt64 {
+ t.Errorf("stepInt64(MaxInt64, decrease) = %d, want MaxInt64", got)
+ }
+ if got := stepInt64(5, false); got != 4 {
+ t.Errorf("stepInt64(5, decrease) = %d, want 4", got)
+ }
+ if got := stepInt64(5, true); got != 6 {
+ t.Errorf("stepInt64(5, increase) = %d, want 6", got)
+ }
+}
+
+func TestSortableInt64OrdersLikeSigned(t *testing.T) {
+ values := []int64{math.MinInt64, -1, 0, 1, math.MaxInt64}
+ for outer := 0; outer+1 < len(values); outer++ {
+ if sortableInt64(values[outer]) >= sortableInt64(values[outer+1]) {
+ t.Errorf("sortableInt64(%d) >= sortableInt64(%d)", values[outer], values[outer+1])
+ }
+ }
+}
diff --git a/pkg/index/nativeadapter/searcher.go b/pkg/index/nativeadapter/searcher.go
index b93430ebc..dda35e075 100644
--- a/pkg/index/nativeadapter/searcher.go
+++ b/pkg/index/nativeadapter/searcher.go
@@ -40,6 +40,8 @@ const nativeSeriesField = "_series_id"
// SearcherOptions supplies the mandatory native budgets for dictionary
// expansion and candidate materialization. A zero MaxTerms is rejected by
// NewSearcher rather than silently turning MatchField into an unbounded walk.
+//
+//nolint:govet // option fields are grouped by construction-time vs per-query role, not padding.
type SearcherOptions struct {
MaxTerms uint64
MaxCandidates uint64
@@ -50,6 +52,10 @@ type SearcherOptions struct {
// PersistInterval spaces background persists when AsyncPersistence is
// set; see native.OwnerOptions.PersistInterval.
PersistInterval time.Duration
+ // TimeMetrics receives time-range pruning counts from every query the
+ // owner NewStore creates serves; see native.OwnerOptions.TimeMetrics. It
+ // is read only by NewStore, not by NewSearcher.
+ TimeMetrics native.TimeMetrics
}
// Searcher binds one request context and one immutable native ReadView. The
diff --git a/pkg/index/nativeadapter/store.go b/pkg/index/nativeadapter/store.go
index 2ff4ef314..65b428ebd 100644
--- a/pkg/index/nativeadapter/store.go
+++ b/pkg/index/nativeadapter/store.go
@@ -65,6 +65,7 @@ func NewStore(path string, lease RootLease, options SearcherOptions) (*Store, er
// from after a rollback. Property's own native store stays without
// this -- its file rollback is already proven without it.
IdentifierDocValues: true,
+ TimeMetrics: options.TimeMetrics,
}
if options.AsyncPersistence {
ownerOptions.PersistInterval = options.PersistInterval
diff --git a/pkg/test/eventually.go b/pkg/test/eventually.go
index bdf0a2ad6..f7707932d 100644
--- a/pkg/test/eventually.go
+++ b/pkg/test/eventually.go
@@ -18,6 +18,7 @@
package test
import (
+ "errors"
"reflect"
"sync/atomic"
"time"
@@ -126,6 +127,35 @@ func countEvaluations(actual interface{}) (interface{}, *int32) {
return probe, attempts
}
+// EventuallyErr runs fn repeatedly (gomega.Eventually bounded by flags.EventuallyTimeout, or
+// the first entry of intervals if supplied, mirroring gomega.Eventually's own (timeout) /
+// (timeout, polling) signature) and converts the result into a plain error instead of
+// failing the enclosing Ginkgo spec. It is for plain Go helper functions that compose
+// several fallible steps — outside any It/By closure — and need to retry a step that can
+// transiently fail under eventual consistency (a schema-cache lag, a topology race, ...)
+// while still returning an error their caller can wrap with step-level context.
+//
+// fn receives a scoped Gomega; use it (not the package-level gomega.Expect/Ω) so a failed
+// assertion inside fn is treated as "not yet converged" and retried, rather than aborting
+// the whole call stack. Call gomega.StopTrying(...).Now() from within fn to end retrying
+// immediately on a failure that must not be repeated — e.g. a partially-accepted write
+// batch, where resending would duplicate the already-accepted rows.
+func EventuallyErr(fn func(g gomega.Gomega), intervals ...interface{}) error {
+ if len(intervals) == 0 {
+ intervals = []interface{}{flags.EventuallyTimeout}
+ }
+ var failure string
+ localG := gomega.NewGomega(func(message string, _ ...int) {
+ if failure == "" {
+ failure = message
+ }
+ })
+ if localG.Eventually(fn, intervals...).Should(gomega.Succeed()) {
+ return nil
+ }
+ return errors.New(failure)
+}
+
func (a ECAssertion) eventuallyIntervals() []interface{} {
timeout := a.eventuallyTimeout
if timeout == nil {
diff --git a/pkg/test/eventually_test.go b/pkg/test/eventually_test.go
index e25dfa23e..c8d0c8bcf 100644
--- a/pkg/test/eventually_test.go
+++ b/pkg/test/eventually_test.go
@@ -60,3 +60,57 @@ func TestEventuallyConsistentlyHoldsWindowAfterRetriedConvergence(t *testing.T)
t.Fatalf("retried convergence must hold the full stability window %v, took %v", flags.StabilityTimeout, elapsed)
}
}
+
+func TestEventuallyErrSucceedsWithoutRetrying(t *testing.T) {
+ var calls int32
+ err := test.EventuallyErr(func(_ gomega.Gomega) {
+ atomic.AddInt32(&calls, 1)
+ }, time.Second, 10*time.Millisecond)
+ if err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+ if got := atomic.LoadInt32(&calls); got != 1 {
+ t.Fatalf("expected exactly one evaluation on first-try success, got %d", got)
+ }
+}
+
+func TestEventuallyErrRetriesUntilConverged(t *testing.T) {
+ var calls int32
+ err := test.EventuallyErr(func(g gomega.Gomega) {
+ g.Expect(atomic.AddInt32(&calls, 1)).To(gomega.BeNumerically(">=", int32(3)))
+ }, time.Second, 10*time.Millisecond)
+ if err != nil {
+ t.Fatalf("expected no error after convergence, got %v", err)
+ }
+ if got := atomic.LoadInt32(&calls); got < 3 {
+ t.Fatalf("expected at least 3 evaluations before convergence, got %d", got)
+ }
+}
+
+func TestEventuallyErrReturnsErrorWithoutFailingTheTestOnTimeout(t *testing.T) {
+ err := test.EventuallyErr(func(g gomega.Gomega) {
+ g.Expect(false).To(gomega.BeTrue(), "deliberately never converges")
+ }, 100*time.Millisecond, 10*time.Millisecond)
+ if err == nil {
+ t.Fatal("expected an error once the timeout elapses")
+ }
+}
+
+func TestEventuallyErrStopsTryingImmediately(t *testing.T) {
+ var calls int32
+ start := time.Now()
+ err := test.EventuallyErr(func(_ gomega.Gomega) {
+ atomic.AddInt32(&calls, 1)
+ gomega.StopTrying("partial success cannot be safely retried").Now()
+ }, time.Second, 10*time.Millisecond)
+ elapsed := time.Since(start)
+ if err == nil {
+ t.Fatal("expected StopTrying to surface as an error")
+ }
+ if got := atomic.LoadInt32(&calls); got != 1 {
+ t.Fatalf("StopTrying must stop after the first evaluation, got %d", got)
+ }
+ if elapsed >= time.Second {
+ t.Fatalf("StopTrying must not wait out the full timeout, took %v", elapsed)
+ }
+}
diff --git a/test/cases/measure/data/data.go b/test/cases/measure/data/data.go
index f3fa73698..3c694f2ff 100644
--- a/test/cases/measure/data/data.go
+++ b/test/cases/measure/data/data.go
@@ -295,31 +295,40 @@ func Write(conn *grpclib.ClientConn, name, group, dataFile string,
func WriteWithAuth(conn *grpclib.ClientConn, name, group, dataFile string,
baseTime time.Time, interval time.Duration, username, password string,
) {
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
- ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("username", username, "password", password))
+ getCtx, getCancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer getCancel()
+ getCtx = metadata.NewOutgoingContext(getCtx, metadata.Pairs("username", username, "password", password))
meta := &commonv1.Metadata{
Name: name,
Group: group,
}
schemaClient := databasev1.NewMeasureRegistryServiceClient(conn)
- resp, err := schemaClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: meta})
+ resp, err := schemaClient.Get(getCtx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: meta})
gm.Expect(err).NotTo(gm.HaveOccurred())
meta = resp.GetMeasure().GetMetadata()
c := measurev1.NewMeasureServiceClient(conn)
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- loadData(meta, writeClient, dataFile, baseTime, interval)
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ // The registry Get above reads the schema server, not this liaison's own write
+ // cache, which can still lag it for a moment; a write that races that lag is
+ // refused by status rather than a stream error. Retry the whole attempt (a fresh
+ // stream, the send, and the drain) together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("username", username, "password", password))
+ writeClient, writeErr := c.Write(ctx)
+ innerGm.Expect(writeErr).NotTo(gm.HaveOccurred())
+ loadData(meta, writeClient, dataFile, baseTime, interval)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
-func expectWriteSucceeded(writeClient measurev1.MeasureService_WriteClient) {
- // Setup already waited on SchemaBarrier AwaitSchemaApplied; a NOT_FOUND
- // here means the barrier contract was violated and must fail loudly.
- // Callers must open Write with a deadline so a stalled Recv cannot hang forever.
+// collectWriteStatuses drains a write stream to EOF and returns the status string of every
+// response, in receive order. Callers must open Write with a deadline so a stalled Recv
+// cannot hang forever.
+func collectWriteStatuses(writeClient measurev1.MeasureService_WriteClient) []string {
var statuses []string
for {
ack, recvErr := writeClient.Recv()
@@ -329,10 +338,26 @@ func expectWriteSucceeded(writeClient measurev1.MeasureService_WriteClient) {
gm.Expect(recvErr).NotTo(gm.HaveOccurred())
statuses = append(statuses, ack.GetStatus())
}
- gm.Expect(statuses).NotTo(gm.BeEmpty())
+ return statuses
+}
+
+// expectWriteSucceeded drains writeClient and asserts every row in this write attempt
+// succeeded. A schema barrier is only a hint under eventual consistency: the liaison's
+// own write cache can still lag it for a moment, so a write that races that lag is
+// refused by status (e.g. STATUS_NOT_FOUND), not a stream error. Call this from inside a
+// gm.Eventually so that refusal is retried. A measure row is keyed by series and timestamp
+// with the latest version winning, so resending a partly accepted batch is idempotent;
+// only a refusal that persists past flags.EventuallyTimeout is a real failure.
+func expectWriteSucceeded(innerGm gm.Gomega, writeClient measurev1.MeasureService_WriteClient) {
+ statuses := collectWriteStatuses(writeClient)
+ innerGm.Expect(statuses).NotTo(gm.BeEmpty())
+ succeeded := 0
for _, writeStatus := range statuses {
- gm.Expect(writeStatus).To(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
}
+ innerGm.Expect(succeeded).To(gm.Equal(len(statuses)), "measure write rejected; statuses=%v", statuses)
}
// WriteOnly write data into the server and return the write client.
@@ -366,92 +391,104 @@ type WriteSpec struct {
// WriteWithSpec writes data using multiple data_point_specs to specify tag and field names.
func WriteWithSpec(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration, writeSpecs ...WriteSpec) {
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
schemaClient := databasev1.NewMeasureRegistryServiceClient(conn)
c := measurev1.NewMeasureServiceClient(conn)
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- var currentMd *commonv1.Metadata
- currentTime := baseTime
- for _, ws := range writeSpecs {
- needMetadataUpdate := false
- if ws.Metadata != nil {
- resp, getErr := schemaClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: ws.Metadata})
- gm.Expect(getErr).NotTo(gm.HaveOccurred())
- currentMd = resp.GetMeasure().GetMetadata()
- needMetadataUpdate = true
- }
- gm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
-
- var templates []interface{}
- content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
- gm.Expect(readErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
-
- isFirstForSpec := true
- for i, template := range templates {
- rawDataPointValue, errMarshal := json.Marshal(template)
- gm.Expect(errMarshal).ShouldNot(gm.HaveOccurred())
- dataPointValue := &measurev1.DataPointValue{}
- gm.Expect(protojson.Unmarshal(rawDataPointValue, dataPointValue)).ShouldNot(gm.HaveOccurred())
- dataPointValue.Timestamp = timestamppb.New(currentTime.Add(time.Duration(i) * interval))
- req := &measurev1.WriteRequest{DataPoint: dataPointValue, MessageId: uint64(time.Now().UnixNano())}
- if isFirstForSpec {
- if ws.Spec != nil {
- req.DataPointSpec = ws.Spec
- }
- if needMetadataUpdate {
- req.Metadata = currentMd
+ // The liaison's write cache can still lag a schema barrier, refusing a write by
+ // status rather than a stream error, so the whole attempt (a fresh stream, every
+ // schema Get and Send, and the drain) is retried together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, err := c.Write(ctx)
+ innerGm.Expect(err).NotTo(gm.HaveOccurred())
+
+ var currentMd *commonv1.Metadata
+ currentTime := baseTime
+ for _, ws := range writeSpecs {
+ needMetadataUpdate := false
+ if ws.Metadata != nil {
+ resp, getErr := schemaClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: ws.Metadata})
+ innerGm.Expect(getErr).NotTo(gm.HaveOccurred())
+ currentMd = resp.GetMeasure().GetMetadata()
+ needMetadataUpdate = true
+ }
+ innerGm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
+
+ var templates []interface{}
+ content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
+ innerGm.Expect(readErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
+
+ isFirstForSpec := true
+ for i, template := range templates {
+ rawDataPointValue, errMarshal := json.Marshal(template)
+ innerGm.Expect(errMarshal).ShouldNot(gm.HaveOccurred())
+ dataPointValue := &measurev1.DataPointValue{}
+ innerGm.Expect(protojson.Unmarshal(rawDataPointValue, dataPointValue)).ShouldNot(gm.HaveOccurred())
+ dataPointValue.Timestamp = timestamppb.New(currentTime.Add(time.Duration(i) * interval))
+ req := &measurev1.WriteRequest{DataPoint: dataPointValue, MessageId: uint64(time.Now().UnixNano())}
+ if isFirstForSpec {
+ if ws.Spec != nil {
+ req.DataPointSpec = ws.Spec
+ }
+ if needMetadataUpdate {
+ req.Metadata = currentMd
+ }
+ isFirstForSpec = false
}
- isFirstForSpec = false
+ innerGm.Expect(writeClient.Send(req)).Should(gm.Succeed())
}
- gm.Expect(writeClient.Send(req)).Should(gm.Succeed())
+ currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
}
- currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
- }
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// WriteMixed writes measure data in schema order first, and then in spec order.
func WriteMixed(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration, writeSpecs ...WriteSpec) {
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
schemaClient := databasev1.NewMeasureRegistryServiceClient(conn)
c := measurev1.NewMeasureServiceClient(conn)
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- var currentMd *commonv1.Metadata
- currentTime := baseTime
- for idx, ws := range writeSpecs {
- if ws.Metadata != nil {
- resp, getErr := schemaClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: ws.Metadata})
- gm.Expect(getErr).NotTo(gm.HaveOccurred())
- currentMd = resp.GetMeasure().GetMetadata()
- }
- gm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
-
- var templates []interface{}
- content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
- gm.Expect(readErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
-
- if idx == 0 {
- loadData(currentMd, writeClient, ws.DataFile, currentTime.Add(time.Duration(len(templates)-1)*interval), interval)
- } else {
- var mdToSend *commonv1.Metadata
+ // The liaison's write cache can still lag a schema barrier, refusing a write by
+ // status rather than a stream error, so the whole attempt (a fresh stream, every
+ // schema Get and loadData/loadDataWithSpec call, and the drain) is retried together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, err := c.Write(ctx)
+ innerGm.Expect(err).NotTo(gm.HaveOccurred())
+
+ var currentMd *commonv1.Metadata
+ currentTime := baseTime
+ for idx, ws := range writeSpecs {
if ws.Metadata != nil {
- mdToSend = currentMd
+ resp, getErr := schemaClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{Metadata: ws.Metadata})
+ innerGm.Expect(getErr).NotTo(gm.HaveOccurred())
+ currentMd = resp.GetMeasure().GetMetadata()
}
- loadDataWithSpec(mdToSend, writeClient, ws.DataFile, currentTime.Add(time.Duration(len(templates)-1)*interval), interval, ws.Spec)
+ innerGm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
+
+ var templates []interface{}
+ content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
+ innerGm.Expect(readErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
+
+ if idx == 0 {
+ loadData(currentMd, writeClient, ws.DataFile, currentTime.Add(time.Duration(len(templates)-1)*interval), interval)
+ } else {
+ var mdToSend *commonv1.Metadata
+ if ws.Metadata != nil {
+ mdToSend = currentMd
+ }
+ loadDataWithSpec(mdToSend, writeClient, ws.DataFile, currentTime.Add(time.Duration(len(templates)-1)*interval), interval, ws.Spec)
+ }
+ currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
}
- currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
- }
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
diff --git a/test/cases/property/data/data.go b/test/cases/property/data/data.go
index 9fbb4ce15..9871eb17f 100644
--- a/test/cases/property/data/data.go
+++ b/test/cases/property/data/data.go
@@ -41,6 +41,7 @@ import (
metadatapkg "github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/pkg/bydbql"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
)
@@ -204,6 +205,12 @@ func WriteToGroup(conn *grpclib.ClientConn, name, group, fileName string) {
gm.Expect(protojson.Unmarshal(content, &request)).ShouldNot(gm.HaveOccurred())
request.Property.Metadata = metadata
- _, err = c.Apply(ctx, &request)
- gm.Expect(err).ShouldNot(gm.HaveOccurred())
+ // A just-created group/property schema reaches the liaison's in-memory repositories
+ // asynchronously via the schema watch, so an Apply issued right after can transiently
+ // fail. Eventually rides out that propagation window.
+ gm.Eventually(func() error {
+ var applyErr error
+ _, applyErr = c.Apply(ctx, &request)
+ return applyErr
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
diff --git a/test/cases/schema/apply_rollback.go b/test/cases/schema/apply_rollback.go
index e37ccc50c..d8bcf3f24 100644
--- a/test/cases/schema/apply_rollback.go
+++ b/test/cases/schema/apply_rollback.go
@@ -31,6 +31,7 @@ import (
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
)
// arMeasureGroup creates a CATALOG_MEASURE group suitable for apply-rollback tests.
@@ -399,12 +400,15 @@ var _ = g.Describe("Schema apply rollback", func() {
"ListMeasure must contain no user-created entries after deletion")
g.By("Verifying write to deleted measure fails or returns non-SUCCEED status")
- writeStatus, writeErr := sendSingleMeasureWrite(ctx, clients.MeasureWriteClient, groupName, measureName, 0)
- // The measure entity is no longer in the routing cache; write must fail.
- if writeErr == nil {
- gm.Expect(writeStatus).ShouldNot(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "write to deleted measure must not return STATUS_SUCCEED")
- }
+ // AwaitDeleted confirms only the cluster watermark; this liaison's own entity cache can
+ // still hold the measure's locator for a moment after that, in which case an immediate
+ // write transiently returns STATUS_SUCCEED (a stray row in a group this spec deletes
+ // right after, so nothing later reads it) instead of failing. Retry until the cache
+ // catches up and the write genuinely fails or returns a non-SUCCEED status.
+ gm.Eventually(func() (string, error) {
+ return sendSingleMeasureWrite(ctx, clients.MeasureWriteClient, groupName, measureName, 0)
+ }, flags.EventuallyTimeout).ShouldNot(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
+ "write to deleted measure must not return STATUS_SUCCEED")
_, _ = clients.GroupClient.Delete(ctx, &databasev1.GroupRegistryServiceDeleteRequest{Group: groupName})
})
@@ -473,9 +477,15 @@ var _ = g.Describe("Schema apply rollback", func() {
"ListMeasure must return exactly one user-created entry after recreate")
g.By("Write with stale R1 must return STATUS_EXPIRED_SCHEMA")
- writeStatus, writeErr := sendSingleMeasureWrite(ctx, clients.MeasureWriteClient, groupName, measureName, r1)
- gm.Expect(writeErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
+ // AwaitRevision(r2) confirms only the cluster watermark. Before this liaison's own
+ // cache has replayed both the delete and the recreate, a write naming R1 can see
+ // NOT_FOUND (delete replayed, recreate not yet) or even STATUS_SUCCEED (delete not yet
+ // replayed, so R1 still looks current) — a stray row in a group this spec deletes right
+ // after, so nothing later reads it. Retry until the cache has caught up to R2 and
+ // genuinely reports R1 as stale.
+ gm.Eventually(func() (string, error) {
+ return sendSingleMeasureWrite(ctx, clients.MeasureWriteClient, groupName, measureName, r1)
+ }, flags.EventuallyTimeout).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
"write with stale ModRevision R1 must return STATUS_EXPIRED_SCHEMA after recreate")
_, _ = clients.GroupClient.Delete(ctx, &databasev1.GroupRegistryServiceDeleteRequest{Group: groupName})
diff --git a/test/cases/schema/clamp.go b/test/cases/schema/clamp.go
index e20017a1b..6b88bbdd7 100644
--- a/test/cases/schema/clamp.go
+++ b/test/cases/schema/clamp.go
@@ -32,6 +32,7 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -103,9 +104,20 @@ var _ = g.Describe("Schema time-range clamp", func() {
g.By("Querying with a range that ends far in the past (before schema creation)")
epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
- resp, queryErr := sendStreamQueryWithRange(ctx, clients.StreamWriteClient, groupName, streamName,
- epoch, epoch.Add(time.Millisecond), streamRev)
- gm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ // AwaitApplied confirms the stream key is present, but a schema revision just applied
+ // can still lag this liaison's own cache for a moment; until it catches up, the
+ // query-gate short-circuits with an empty response before the clamp logic runs, which
+ // would make the emptiness below pass without exercising the clamp. Retry until the
+ // gate reports groupName as SUCCEED.
+ var resp *streamv1.QueryResponse
+ gm.Eventually(func(innerGm gm.Gomega) {
+ var queryErr error
+ resp, queryErr = sendStreamQueryWithRange(ctx, clients.StreamWriteClient, groupName, streamName,
+ epoch, epoch.Add(time.Millisecond), streamRev)
+ innerGm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(resp.GetGroupStatuses()[groupName]).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED),
+ "schema gate for %s must report SUCCEED before the clamp assertion is meaningful", groupName)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
gm.Expect(resp.GetElements()).Should(gm.BeEmpty(),
"clamped Begin (≈ CreatedAt) > End (epoch+1ms) must produce an empty response")
@@ -191,22 +203,35 @@ var _ = g.Describe("Schema time-range clamp", func() {
g.By("Querying both groups with a range that ends far in the past")
epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
- resp, queryErr := clients.StreamWriteClient.Query(ctx, &streamv1.QueryRequest{
- Groups: []string{group1, group2},
- Name: streamName,
- TimeRange: &modelv1.TimeRange{
- Begin: timestamppb.New(epoch),
- End: timestamppb.New(epoch.Add(time.Millisecond)),
- },
- Limit: 10,
- GroupModRevisions: streamRevs,
- Projection: &modelv1.TagProjection{
- TagFamilies: []*modelv1.TagProjection_TagFamily{
- {Name: "default", Tags: []string{"svc"}},
+ // AwaitApplied confirms both stream keys are present, but a just-applied revision can
+ // still lag either liaison's own cache for a moment; until both catch up, the
+ // query-gate short-circuits with an empty response before the clamp logic runs, which
+ // would make the emptiness below pass without exercising the clamp. Retry until the
+ // gate reports both groups as SUCCEED.
+ var resp *streamv1.QueryResponse
+ gm.Eventually(func(innerGm gm.Gomega) {
+ var queryErr error
+ resp, queryErr = clients.StreamWriteClient.Query(ctx, &streamv1.QueryRequest{
+ Groups: []string{group1, group2},
+ Name: streamName,
+ TimeRange: &modelv1.TimeRange{
+ Begin: timestamppb.New(epoch),
+ End: timestamppb.New(epoch.Add(time.Millisecond)),
},
- },
- })
- gm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ Limit: 10,
+ GroupModRevisions: streamRevs,
+ Projection: &modelv1.TagProjection{
+ TagFamilies: []*modelv1.TagProjection_TagFamily{
+ {Name: "default", Tags: []string{"svc"}},
+ },
+ },
+ })
+ innerGm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ for _, groupName := range []string{group1, group2} {
+ innerGm.Expect(resp.GetGroupStatuses()[groupName]).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED),
+ "schema gate for %s must report SUCCEED before the clamp assertion is meaningful", groupName)
+ }
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
gm.Expect(resp.GetElements()).Should(gm.BeEmpty(),
"max CreatedAt across both groups > epoch+1ms so the clamped range must be empty")
@@ -249,10 +274,17 @@ var _ = g.Describe("Schema time-range clamp", func() {
Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "h1"}},
}},
}}
- writeStatus, writeErr := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, group1, measureName, tData1, r1, hostTags)
- gm.Expect(writeErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "write at T_data1 must return STATUS_SUCCEED")
+ // AwaitRevision is a hint under eventual consistency: another event (such as
+ // the _top_n_result measure each node creates for the new group) can carry the
+ // liaison's watermark past r1 before clamp_measure reaches its write cache, so
+ // the first write can be refused with STATUS_NOT_FOUND. A refused write stores
+ // nothing, so retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus, writeErr := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, group1, measureName, tData1, r1, hostTags)
+ innerGm.Expect(writeErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
+ "write at T_data1 must return STATUS_SUCCEED")
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Sanity-checking that querying group1 alone returns the datum (legacy unclamped baseline — pass modRevision=0)")
// TWO distinct races have to settle before this query can succeed, and the
diff --git a/test/cases/schema/deletion.go b/test/cases/schema/deletion.go
index 7313e75a0..192730c7a 100644
--- a/test/cases/schema/deletion.go
+++ b/test/cases/schema/deletion.go
@@ -38,6 +38,8 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
)
@@ -75,7 +77,7 @@ var _ = g.Describe("Schema deletion", func() {
gm.Expect(err).ShouldNot(gm.HaveOccurred())
g.By("Creating measure schema")
- err = createMeasureSchema(ctx, clients.MeasureRegClient, groupName, measureName)
+ err = createMeasureSchema(ctx, clients, groupName, measureName)
gm.Expect(err).ShouldNot(gm.HaveOccurred())
g.By("Verifying measure deletion")
@@ -169,7 +171,7 @@ func VerifyMeasureDeletion(ctx context.Context, clients *Clients, groupName, mea
// Step 4 & 5: Write to different measure and verify
secondMeasureName := measureName + "_second"
- if err := createMeasureSchema(ctx, clients.MeasureRegClient, groupName, secondMeasureName); err != nil {
+ if err := createMeasureSchema(ctx, clients, groupName, secondMeasureName); err != nil {
return fmt.Errorf("step 4 failed - create second measure: %w", err)
}
for i := 0; i < 20; i++ {
@@ -287,8 +289,8 @@ func VerifyTraceDeletion(ctx context.Context, clients *Clients, groupName, trace
// Helper functions.
-func createMeasureSchema(ctx context.Context, client databasev1.MeasureRegistryServiceClient, groupName, measureName string) error {
- _, err := client.Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{
+func createMeasureSchema(ctx context.Context, clients *Clients, groupName, measureName string) error {
+ resp, err := clients.MeasureRegClient.Create(ctx, &databasev1.MeasureRegistryServiceCreateRequest{
Measure: &databasev1.Measure{
Metadata: &commonv1.Metadata{Name: measureName, Group: groupName},
Entity: &databasev1.Entity{TagNames: []string{"id"}},
@@ -304,8 +306,14 @@ func createMeasureSchema(ctx context.Context, client databasev1.MeasureRegistryS
}},
},
})
- time.Sleep(2 * time.Second)
- return err
+ if err != nil {
+ return err
+ }
+ // AwaitRevision is a hint under eventual consistency: it confirms the cluster watermark,
+ // not that this liaison's own entity cache has applied the schema yet. The write helpers
+ // below retry a refused write for that residual lag, so the barrier only needs to
+ // replace the blind sleep this used to rely on for the common case.
+ return clients.AwaitRevision(ctx, resp.GetModRevision(), flags.EventuallyTimeout)
}
func createStreamSchema(ctx context.Context, clients *Clients, groupName, streamName string) error {
@@ -333,7 +341,7 @@ func createStreamSchema(ctx context.Context, clients *Clients, groupName, stream
if err != nil {
return err
}
- _, err = clients.IndexRuleBindingClient.Create(ctx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
+ bindingResp, err := clients.IndexRuleBindingClient.Create(ctx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
IndexRuleBinding: &databasev1.IndexRuleBinding{
Metadata: &commonv1.Metadata{Name: streamName + "_binding", Group: groupName},
Rules: []string{indexRuleName},
@@ -348,8 +356,12 @@ func createStreamSchema(ctx context.Context, clients *Clients, groupName, stream
if err != nil {
return err
}
- time.Sleep(2 * time.Second)
- return nil
+ // The index rule binding is created last, so its mod_revision is the highest of the
+ // three events this function creates — revisions are a single cluster-wide counter, so
+ // awaiting it is a (hint-only) proxy for all three having reached the watermark. The
+ // write helpers below retry a refused write for the residual per-liaison lag this
+ // barrier does not close.
+ return clients.AwaitRevision(ctx, bindingResp.GetModRevision(), flags.EventuallyTimeout)
}
func createTraceSchema(ctx context.Context, clients *Clients, groupName, traceName string) error {
@@ -382,7 +394,7 @@ func createTraceSchema(ctx context.Context, clients *Clients, groupName, traceNa
if err != nil {
return err
}
- _, err = clients.IndexRuleBindingClient.Create(ctx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
+ bindingResp, err := clients.IndexRuleBindingClient.Create(ctx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
IndexRuleBinding: &databasev1.IndexRuleBinding{
Metadata: &commonv1.Metadata{Name: traceName + "_binding", Group: groupName},
Rules: []string{indexRuleName},
@@ -397,14 +409,20 @@ func createTraceSchema(ctx context.Context, clients *Clients, groupName, traceNa
if err != nil {
return err
}
- time.Sleep(2 * time.Second)
- return nil
+ // The index rule binding is created last, so its mod_revision is the highest of the
+ // three events this function creates — revisions are a single cluster-wide counter, so
+ // awaiting it is a (hint-only) proxy for all three having reached the watermark. The
+ // write helpers below retry a refused write for the residual per-liaison lag this
+ // barrier does not close.
+ return clients.AwaitRevision(ctx, bindingResp.GetModRevision(), flags.EventuallyTimeout)
}
-func writeMeasureData(ctx context.Context, client measurev1.MeasureServiceClient, groupName, measureName string, count int) error {
+// sendMeasureDataBatch writes count distinct measure rows on one write stream and returns
+// the status string of every response, in send order.
+func sendMeasureDataBatch(ctx context.Context, client measurev1.MeasureServiceClient, groupName, measureName string, count int) ([]string, error) {
writeClient, err := client.Write(ctx)
if err != nil {
- return err
+ return nil, err
}
metadata := &commonv1.Metadata{Name: measureName, Group: groupName}
baseTime := time.Now().Truncate(time.Millisecond)
@@ -424,31 +442,53 @@ func writeMeasureData(ctx context.Context, client measurev1.MeasureServiceClient
},
MessageId: uint64(time.Now().UnixNano() + int64(idx)),
}); err != nil {
- return err
+ return nil, err
}
}
if err := writeClient.CloseSend(); err != nil {
- return err
+ return nil, err
}
+ var statuses []string
for {
resp, recvErr := writeClient.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
if recvErr != nil {
- return recvErr
- }
- if resp != nil && resp.Status != modelv1.Status_STATUS_SUCCEED.String() {
- return fmt.Errorf("write failed with status: %s", resp.Status)
+ return statuses, recvErr
}
+ statuses = append(statuses, resp.GetStatus())
}
- return nil
+ return statuses, nil
}
-func writeStreamData(ctx context.Context, client streamv1.StreamServiceClient, groupName, streamName string, count int) error {
+// writeMeasureData retries sendMeasureDataBatch until every row is accepted. Rows can be
+// refused by status until the liaison's schema cache catches up. A measure row is keyed by
+// series and timestamp with the latest version winning, so resending the batch with the same
+// values is idempotent: any refusal, including a partly accepted batch, is retried within
+// flags.EventuallyTimeout.
+func writeMeasureData(ctx context.Context, client measurev1.MeasureServiceClient, groupName, measureName string, count int) error {
+ return test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendMeasureDataBatch(ctx, client, groupName, measureName, count)
+ innerGm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(statuses).Should(gm.HaveLen(count), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ innerGm.Expect(succeeded).Should(gm.Equal(count),
+ "measure write to %s/%s rejected; statuses=%v", groupName, measureName, statuses)
+ })
+}
+
+// sendStreamDataBatch writes count distinct stream rows on one write stream and returns the
+// status string of every response, in send order.
+func sendStreamDataBatch(ctx context.Context, client streamv1.StreamServiceClient, groupName, streamName string, count int) ([]string, error) {
writeClient, err := client.Write(ctx)
if err != nil {
- return err
+ return nil, err
}
metadata := &commonv1.Metadata{Name: streamName, Group: groupName}
baseTime := time.Now().Truncate(time.Millisecond)
@@ -466,31 +506,56 @@ func writeStreamData(ctx context.Context, client streamv1.StreamServiceClient, g
},
MessageId: uint64(time.Now().UnixNano() + int64(idx)),
}); err != nil {
- return err
+ return nil, err
}
}
if err := writeClient.CloseSend(); err != nil {
- return err
+ return nil, err
}
+ var statuses []string
for {
resp, recvErr := writeClient.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
if recvErr != nil {
- return recvErr
- }
- if resp != nil && resp.Status != modelv1.Status_STATUS_SUCCEED.String() {
- return fmt.Errorf("write failed with status: %s", resp.Status)
+ return statuses, recvErr
}
+ statuses = append(statuses, resp.GetStatus())
}
- return nil
+ return statuses, nil
}
-func writeTraceData(ctx context.Context, client tracev1.TraceServiceClient, groupName, traceName string, count int) error {
+// writeStreamData retries sendStreamDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag a schema barrier, and a refused row stores nothing, so
+// repeating the whole batch is safe. If some rows succeed and some are refused, the batch
+// cannot be safely repeated without duplicating the accepted rows, so it stops retrying at
+// once.
+func writeStreamData(ctx context.Context, client streamv1.StreamServiceClient, groupName, streamName string, count int) error {
+ return test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendStreamDataBatch(ctx, client, groupName, streamName, count)
+ innerGm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(statuses).Should(gm.HaveLen(count), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < count {
+ gm.StopTrying(fmt.Sprintf("stream write to %s/%s partially rejected; statuses=%v", groupName, streamName, statuses)).Now()
+ }
+ innerGm.Expect(succeeded).Should(gm.Equal(count),
+ "stream write to %s/%s rejected; statuses=%v", groupName, streamName, statuses)
+ })
+}
+
+// sendTraceDataBatch writes count distinct trace rows on one write stream and returns the
+// status string of every response, in send order.
+func sendTraceDataBatch(ctx context.Context, client tracev1.TraceServiceClient, groupName, traceName string, count int) ([]string, error) {
writeClient, err := client.Write(ctx)
if err != nil {
- return err
+ return nil, err
}
metadata := &commonv1.Metadata{Name: traceName, Group: groupName}
baseTime := time.Now().Truncate(time.Millisecond)
@@ -507,25 +572,48 @@ func writeTraceData(ctx context.Context, client tracev1.TraceServiceClient, grou
Span: []byte(fmt.Sprintf("span_data_%d", idx)),
Version: uint64(idx + 1),
}); err != nil {
- return err
+ return nil, err
}
}
if err := writeClient.CloseSend(); err != nil {
- return err
+ return nil, err
}
+ var statuses []string
for {
resp, recvErr := writeClient.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
if recvErr != nil {
- return recvErr
- }
- if resp != nil && resp.Status != modelv1.Status_STATUS_SUCCEED.String() {
- return fmt.Errorf("write failed with status: %s", resp.Status)
+ return statuses, recvErr
}
+ statuses = append(statuses, resp.GetStatus())
}
- return nil
+ return statuses, nil
+}
+
+// writeTraceData retries sendTraceDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag a schema barrier, and a refused row stores nothing, so
+// repeating the whole batch is safe. If some rows succeed and some are refused, the batch
+// cannot be safely repeated without duplicating the accepted rows, so it stops retrying at
+// once.
+func writeTraceData(ctx context.Context, client tracev1.TraceServiceClient, groupName, traceName string, count int) error {
+ return test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendTraceDataBatch(ctx, client, groupName, traceName, count)
+ innerGm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(statuses).Should(gm.HaveLen(count), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < count {
+ gm.StopTrying(fmt.Sprintf("trace write to %s/%s partially rejected; statuses=%v", groupName, traceName, statuses)).Now()
+ }
+ innerGm.Expect(succeeded).Should(gm.Equal(count),
+ "trace write to %s/%s rejected; statuses=%v", groupName, traceName, statuses)
+ })
}
func verifyMeasureDeletionEffects(ctx context.Context, clients *Clients, groupName, measureName string) error {
@@ -558,8 +646,22 @@ func verifyMeasureDeletionEffects(ctx context.Context, clients *Clients, groupNa
}
}
- if err := writeMeasureData(ctx, clients.MeasureWriteClient, groupName, measureName, 1); err == nil {
- return fmt.Errorf("write to deleted measure should fail")
+ // AwaitDeleted confirms only the cluster watermark; this liaison's own entity cache can
+ // still hold the measure's locator for a moment after that, in which case an immediate
+ // write can transiently succeed instead of failing. Retry until the cache has caught up
+ // and the write genuinely fails or returns a non-SUCCEED status.
+ if err := clients.AwaitDeleted(ctx, []string{fmt.Sprintf("measure:%s/%s", groupName, measureName)}, flags.EventuallyTimeout); err != nil {
+ return fmt.Errorf("await measure deletion: %w", err)
+ }
+ convergedErr := test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendMeasureDataBatch(ctx, clients.MeasureWriteClient, groupName, measureName, 1)
+ if sendErr != nil {
+ return
+ }
+ innerGm.Expect(statuses).ShouldNot(gm.ContainElement(modelv1.Status_STATUS_SUCCEED.String()))
+ })
+ if convergedErr != nil {
+ return fmt.Errorf("write to deleted measure should fail: %w", convergedErr)
}
return nil
@@ -595,8 +697,22 @@ func verifyStreamDeletionEffects(ctx context.Context, clients *Clients, groupNam
}
}
- if err := writeStreamData(ctx, clients.StreamWriteClient, groupName, streamName, 1); err == nil {
- return fmt.Errorf("write to deleted stream should fail")
+ // AwaitDeleted confirms only the cluster watermark; this liaison's own entity cache can
+ // still hold the stream's locator for a moment after that, in which case an immediate
+ // write can transiently succeed instead of failing. Retry until the cache has caught up
+ // and the write genuinely fails or returns a non-SUCCEED status.
+ if err := clients.AwaitDeleted(ctx, []string{fmt.Sprintf("stream:%s/%s", groupName, streamName)}, flags.EventuallyTimeout); err != nil {
+ return fmt.Errorf("await stream deletion: %w", err)
+ }
+ convergedErr := test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendStreamDataBatch(ctx, clients.StreamWriteClient, groupName, streamName, 1)
+ if sendErr != nil {
+ return
+ }
+ innerGm.Expect(statuses).ShouldNot(gm.ContainElement(modelv1.Status_STATUS_SUCCEED.String()))
+ })
+ if convergedErr != nil {
+ return fmt.Errorf("write to deleted stream should fail: %w", convergedErr)
}
return nil
@@ -654,8 +770,22 @@ func verifyTraceDeletionEffects(ctx context.Context, clients *Clients, groupName
}
}
- if err := writeTraceData(ctx, clients.TraceWriteClient, groupName, traceName, 1); err == nil {
- return fmt.Errorf("write to deleted trace should fail")
+ // AwaitDeleted confirms only the cluster watermark; this liaison's own entity cache can
+ // still hold the trace's locator for a moment after that, in which case an immediate
+ // write can transiently succeed instead of failing. Retry until the cache has caught up
+ // and the write genuinely fails or returns a non-SUCCEED status.
+ if err := clients.AwaitDeleted(ctx, []string{fmt.Sprintf("trace:%s/%s", groupName, traceName)}, flags.EventuallyTimeout); err != nil {
+ return fmt.Errorf("await trace deletion: %w", err)
+ }
+ convergedErr := test.EventuallyErr(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendTraceDataBatch(ctx, clients.TraceWriteClient, groupName, traceName, 1)
+ if sendErr != nil {
+ return
+ }
+ innerGm.Expect(statuses).ShouldNot(gm.ContainElement(modelv1.Status_STATUS_SUCCEED.String()))
+ })
+ if convergedErr != nil {
+ return fmt.Errorf("write to deleted trace should fail: %w", convergedErr)
}
return nil
diff --git a/test/cases/schema/query_gate.go b/test/cases/schema/query_gate.go
index 679152637..e6154354d 100644
--- a/test/cases/schema/query_gate.go
+++ b/test/cases/schema/query_gate.go
@@ -124,9 +124,13 @@ var _ = g.Describe("Schema query gate", func() {
gm.Expect(awaitErr).ShouldNot(gm.HaveOccurred())
g.By("Sending a query with the old ModRevision R1 (stale)")
- st, queryErr := sendStreamQuery(ctx, clients.StreamWriteClient, groupName, streamName, r1)
- gm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(st).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA),
+ // AwaitRevision(r2) confirms only the cluster watermark, not that this liaison's own
+ // cache has applied r2; until it has, the cache still shows r1 as current, so a query
+ // naming r1 passes the gate with STATUS_SUCCEED instead of being refused. Retry until
+ // the cache has caught up and genuinely reports r1 as stale.
+ gm.Eventually(func() (modelv1.Status, error) {
+ return sendStreamQuery(ctx, clients.StreamWriteClient, groupName, streamName, r1)
+ }, flags.EventuallyTimeout).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA),
"query with group ModRevision < cache must return STATUS_EXPIRED_SCHEMA")
_, _ = clients.GroupClient.Delete(ctx, &databasev1.GroupRegistryServiceDeleteRequest{Group: groupName})
@@ -261,25 +265,35 @@ var _ = g.Describe("Schema query gate", func() {
g.By("Querying both groups with group1 stale (R1a) and group2 current (R2)")
now := timestamp.NowMilli()
- resp, queryErr := clients.StreamWriteClient.Query(ctx, &streamv1.QueryRequest{
- Groups: []string{group1, group2},
- Name: streamName,
- TimeRange: &modelv1.TimeRange{
- Begin: timestamppb.New(now.Add(-time.Hour)),
- End: timestamppb.New(now.Add(time.Hour)),
- },
- Limit: 10,
- GroupModRevisions: map[string]int64{
- group1: r1a,
- group2: r2,
- },
- Projection: &modelv1.TagProjection{
- TagFamilies: []*modelv1.TagProjection_TagFamily{
- {Name: "default", Tags: []string{"svc"}},
+ // AwaitRevision(maxRev) confirms only the cluster watermark, not that group1's own
+ // liaison cache has applied r1b; until it has, the cache still shows r1a as current, so
+ // group1 passes the gate with STATUS_SUCCEED (and the query is not short-circuited)
+ // instead of the expected STATUS_EXPIRED_SCHEMA. Retry until group1's cache catches up.
+ var resp *streamv1.QueryResponse
+ gm.Eventually(func(innerGm gm.Gomega) {
+ var queryErr error
+ resp, queryErr = clients.StreamWriteClient.Query(ctx, &streamv1.QueryRequest{
+ Groups: []string{group1, group2},
+ Name: streamName,
+ TimeRange: &modelv1.TimeRange{
+ Begin: timestamppb.New(now.Add(-time.Hour)),
+ End: timestamppb.New(now.Add(time.Hour)),
},
- },
- })
- gm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ Limit: 10,
+ GroupModRevisions: map[string]int64{
+ group1: r1a,
+ group2: r2,
+ },
+ Projection: &modelv1.TagProjection{
+ TagFamilies: []*modelv1.TagProjection_TagFamily{
+ {Name: "default", Tags: []string{"svc"}},
+ },
+ },
+ })
+ innerGm.Expect(queryErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(resp.GetGroupStatuses()[group1]).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA),
+ "group1 with stale R1a must be STATUS_EXPIRED_SCHEMA")
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Asserting group1 status is EXPIRED_SCHEMA and group2 status is SUCCEED")
statuses := resp.GetGroupStatuses()
diff --git a/test/cases/schema/shape_break.go b/test/cases/schema/shape_break.go
index 59ca581bc..67f36bebc 100644
--- a/test/cases/schema/shape_break.go
+++ b/test/cases/schema/shape_break.go
@@ -32,6 +32,7 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -196,10 +197,16 @@ var _ = g.Describe("Schema shape-break rejection", func() {
Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "h1"}},
}},
}}
- writeStatus1, writeErr1 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName, tData1, r1, hostTags1)
- gm.Expect(writeErr1).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus1).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "initial write must return STATUS_SUCCEED")
+ // AwaitRevision is a hint under eventual consistency: another event (such as the
+ // _top_n_result measure each node creates for the new group) can carry the
+ // liaison's watermark past r1 before this write reaches its cache, refusing it
+ // with a status. A refused write stores nothing, so retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus1, writeErr1 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName, tData1, r1, hostTags1)
+ innerGm.Expect(writeErr1).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(writeStatus1).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
+ "initial write must return STATUS_SUCCEED")
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Querying [CreatedAt1, now+1h] — must return exactly 1 data point")
// In distributed mode the write→query path is asynchronous: the
@@ -259,36 +266,53 @@ var _ = g.Describe("Schema shape-break rejection", func() {
"CreatedAt2 must be after T_del")
// Rule 7 clamp: query [T_data1, now+1h] must return empty because T_data1 < CreatedAt2.
- queryResp2, queryErr2 := queryMeasureRange(ctx, clients.MeasureWriteClient, groupName, measureName,
- tData1, time.Now().Add(time.Hour), r2)
- gm.Expect(queryErr2).ShouldNot(gm.HaveOccurred())
+ // AwaitRevision(r2) confirms only the cluster watermark, not that this liaison's own
+ // cache has applied r2; until it has, the query-gate short-circuits with an empty
+ // response before the clamp logic ever runs, which would make the emptiness below pass
+ // without exercising the clamp. Retry until the gate reports groupName as SUCCEED.
+ var queryResp2 *measurev1.QueryResponse
+ gm.Eventually(func(innerGm gm.Gomega) {
+ var queryErr2 error
+ queryResp2, queryErr2 = queryMeasureRange(ctx, clients.MeasureWriteClient, groupName, measureName,
+ tData1, time.Now().Add(time.Hour), r2)
+ innerGm.Expect(queryErr2).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(queryResp2.GetGroupStatuses()[groupName]).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED),
+ "schema gate for %s must report SUCCEED before the clamp assertion is meaningful", groupName)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
gm.Expect(queryResp2.GetDataPoints()).Should(gm.BeEmpty(),
"Rule 7 clamp must hide pre-CreatedAt2 data points")
- // Stale write with R1 must be rejected.
+ // Stale write with R1 must eventually return STATUS_EXPIRED_SCHEMA. Immediately after
+ // AwaitRevision(r2) the liaison's entity cache can still show R1 as current — the
+ // watermark can advance past r2 via another event before r2's own handler runs — in
+ // which case this write is indistinguishable from a current write and is accepted
+ // (STATUS_SUCCEED), landing a stray row under the old R1 shape. Nothing later in this
+ // spec reads that data, so retrying past the stray success is safe.
staleHostTags := []*modelv1.TagFamilyForWrite{{
Tags: []*modelv1.TagValue{{
Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "h1"}},
}},
}}
- writeStatusStale, writeErrStale := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName,
- time.Now(), r1, staleHostTags)
- gm.Expect(writeErrStale).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatusStale).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
+ gm.Eventually(func() (string, error) {
+ return sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName, time.Now(), r1, staleHostTags)
+ }, flags.EventuallyTimeout).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
"write with stale R1 must return STATUS_EXPIRED_SCHEMA")
- // Write with R2 and new tag layout must succeed.
+ // Write with R2 and new tag layout must succeed. Same cache-lag hazard as the R1 write
+ // above, now for r2: a refused write stores nothing, so retrying it is safe.
newTags := []*modelv1.TagFamilyForWrite{{
Tags: []*modelv1.TagValue{
{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "h1"}}},
{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "inst1"}}},
},
}}
- writeStatusNew, writeErrNew := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName,
- time.Now(), r2, newTags)
- gm.Expect(writeErrNew).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatusNew).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "write with R2 and new tag layout must return STATUS_SUCCEED")
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatusNew, writeErrNew := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName,
+ time.Now(), r2, newTags)
+ innerGm.Expect(writeErrNew).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(writeStatusNew).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
+ "write with R2 and new tag layout must return STATUS_SUCCEED")
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
_, _ = clients.GroupClient.Delete(ctx, &databasev1.GroupRegistryServiceDeleteRequest{Group: groupName})
})
@@ -450,9 +474,14 @@ var _ = g.Describe("Schema shape-break rejection", func() {
Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "h1"}},
}},
}}
- writeStatus1, writeErr1 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName, tData1, r1, hostTags)
- gm.Expect(writeErr1).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus1).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // AwaitRevision is a hint under eventual consistency: another event can carry the
+ // liaison's watermark past r1 before this write reaches its cache, refusing it with a
+ // status. A refused write stores nothing, so retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus1, writeErr1 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName, tData1, r1, hostTags)
+ innerGm.Expect(writeErr1).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(writeStatus1).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
// Eventually retry — see the baseline above for the
// distributed write→query visibility race.
@@ -501,19 +530,32 @@ var _ = g.Describe("Schema shape-break rejection", func() {
"recreated measure must preserve original entity")
// Rule 7 clamp: query [T_data1, now+1h] returns empty because T_data1 < CreatedAt2.
- queryResp2, queryErr2 := queryMeasureRange(ctx, clients.MeasureWriteClient, groupName, measureName,
- tData1, time.Now().Add(time.Hour), r2)
- gm.Expect(queryErr2).ShouldNot(gm.HaveOccurred())
+ // AwaitRevision(r2) confirms only the cluster watermark, not that this liaison's own
+ // cache has applied r2; until it has, the query-gate short-circuits with an empty
+ // response before the clamp logic ever runs, which would make the emptiness below pass
+ // without exercising the clamp. Retry until the gate reports groupName as SUCCEED.
+ var queryResp2 *measurev1.QueryResponse
+ gm.Eventually(func(innerGm gm.Gomega) {
+ var queryErr2 error
+ queryResp2, queryErr2 = queryMeasureRange(ctx, clients.MeasureWriteClient, groupName, measureName,
+ tData1, time.Now().Add(time.Hour), r2)
+ innerGm.Expect(queryErr2).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(queryResp2.GetGroupStatuses()[groupName]).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED),
+ "schema gate for %s must report SUCCEED before the clamp assertion is meaningful", groupName)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
gm.Expect(queryResp2.GetDataPoints()).Should(gm.BeEmpty(),
"Rule 7 clamp must hide pre-delete data because T_data1 < CreatedAt2")
- // Write with R2 must succeed.
+ // Write with R2 must succeed. Same cache-lag hazard as the R1 write above, now for r2:
+ // a refused write stores nothing, so retrying it is safe.
tNewWrite := time.Now()
- writeStatus2, writeErr2 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName,
- tNewWrite, r2, hostTags)
- gm.Expect(writeErr2).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus2).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "write with R2 must succeed")
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus2, writeErr2 := sendMeasureWriteAtTime(ctx, clients.MeasureWriteClient, groupName, measureName,
+ tNewWrite, r2, hostTags)
+ innerGm.Expect(writeErr2).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(writeStatus2).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
+ "write with R2 must succeed")
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
// Post-write query [CreatedAt2, now+1h] must return the newly-written point.
// Same write→query visibility race as the baseline.
diff --git a/test/cases/schema/tag_family_move.go b/test/cases/schema/tag_family_move.go
index 0f12d95e6..527b3385b 100644
--- a/test/cases/schema/tag_family_move.go
+++ b/test/cases/schema/tag_family_move.go
@@ -34,6 +34,7 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
)
const (
@@ -982,9 +983,15 @@ var _ = g.Describe("Schema tag family move", func() {
gm.Expect(clients.AwaitRevision(ctx, max(baseRevision, indexRevision), 10*time.Second)).Should(gm.Succeed())
g.By("Writing and querying stream data before the tag is moved")
- writeStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-pre", preMoveTime,
- tagFamilyMoveTags("svc-pre", "host-pre", "region-pre", false), tagFamilyMoveInitialWriteSpec())
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // A single-row write stores nothing when refused, so retrying it after the barrier
+ // above is safe; see the tag-move-in-progress comment below for why a refusal can
+ // still happen here.
+ var writeStatus string
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus = writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-pre", preMoveTime,
+ tagFamilyMoveTags("svc-pre", "host-pre", "region-pre", false), tagFamilyMoveInitialWriteSpec())
+ innerGm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
beforeElements := queryTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, begin, end, false, 1, nil)
expectIndexedMovedTagValue(beforeElements, "svc-pre", "host-pre")
indexedElements := queryTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, begin, end, false, 1, tagFamilyMoveHostCriteria("host-pre"))
@@ -1005,18 +1012,27 @@ var _ = g.Describe("Schema tag family move", func() {
indexMoveRevision := moveTagFamilyMoveStreamIndexes(ctx, clients, groupName, streamName)
g.By("Continuing writes with the old layout while the move is applying")
- writeInMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-in-move", inMoveTime,
- tagFamilyMoveTags("svc-in-move", "host-in-move", "region-in-move", false), tagFamilyMoveInitialWriteSpec())
- gm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // The entity cache can still lag the move while it is applying; a refused write
+ // stores nothing, so retrying is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeInMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-in-move", inMoveTime,
+ tagFamilyMoveTags("svc-in-move", "host-in-move", "region-in-move", false), tagFamilyMoveInitialWriteSpec())
+ innerGm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Waiting for the moved stream tag schema and index binding to be applied")
targetRevision := max(updateResp.GetModRevision(), indexMoveRevision)
gm.Expect(clients.AwaitRevision(ctx, targetRevision, 10*time.Second)).Should(gm.Succeed())
g.By("Writing stream data with the new storage-only host layout")
- writePostMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-post", postMoveTime,
- tagFamilyMoveTags("svc-post", "host-post", "region-post", true), tagFamilyMoveUpdatedWriteSpec())
- gm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // AwaitRevision is a hint under eventual consistency: this liaison's own cache can
+ // still lag the just-applied move for a moment. A refused write stores nothing, so
+ // retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writePostMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-post", postMoveTime,
+ tagFamilyMoveTags("svc-post", "host-post", "region-post", true), tagFamilyMoveUpdatedWriteSpec())
+ innerGm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Verifying the moved stream schema layout")
afterResp, getAfterErr := clients.StreamRegClient.Get(ctx, &databasev1.StreamRegistryServiceGetRequest{
@@ -1065,10 +1081,16 @@ var _ = g.Describe("Schema tag family move", func() {
gm.Expect(clients.AwaitRevision(ctx, max(baseRevision, indexRevision), 10*time.Second)).Should(gm.Succeed())
g.By("Writing and querying stream data before the storage-only tag is moved")
- writeStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-pre", preMoveTime,
- tagFamilyMoveStorageToSearchableTags("svc-storage-pre", "host-storage-pre", "region-storage-pre", false),
- tagFamilyMoveStorageToSearchableInitialWriteSpec())
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // A single-row write stores nothing when refused, so retrying it after the barrier
+ // above is safe; see the tag-move-in-progress comment below for why a refusal can
+ // still happen here.
+ var writeStatus string
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus = writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-pre", preMoveTime,
+ tagFamilyMoveStorageToSearchableTags("svc-storage-pre", "host-storage-pre", "region-storage-pre", false),
+ tagFamilyMoveStorageToSearchableInitialWriteSpec())
+ innerGm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
beforeElements := queryStorageToSearchableMoveStream(ctx, clients.StreamWriteClient, groupName, begin, end, false, 1, nil)
expectStorageOnlyMovedTagValue(beforeElements, "svc-storage-pre", "host-storage-pre")
storageOnlyCriteriaElements := queryStorageToSearchableMoveStream(ctx, clients.StreamWriteClient, groupName, begin, end, false, 1,
@@ -1090,20 +1112,29 @@ var _ = g.Describe("Schema tag family move", func() {
indexMoveRevision := moveTagFamilyMoveStorageToSearchableIndexes(ctx, clients, groupName, streamName)
g.By("Continuing writes with the old storage-only layout while the move is applying")
- writeInMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-in-move", inMoveTime,
- tagFamilyMoveStorageToSearchableTags("svc-storage-in-move", "host-storage-in-move", "region-storage-in-move", false),
- tagFamilyMoveStorageToSearchableInitialWriteSpec())
- gm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // The entity cache can still lag the move while it is applying; a refused write
+ // stores nothing, so retrying is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeInMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-in-move", inMoveTime,
+ tagFamilyMoveStorageToSearchableTags("svc-storage-in-move", "host-storage-in-move", "region-storage-in-move", false),
+ tagFamilyMoveStorageToSearchableInitialWriteSpec())
+ innerGm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Waiting for the moved stream tag schema and index binding to be applied")
targetRevision := max(updateResp.GetModRevision(), indexMoveRevision)
gm.Expect(clients.AwaitRevision(ctx, targetRevision, 10*time.Second)).Should(gm.Succeed())
g.By("Writing stream data with the new searchable host layout")
- writePostMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-post", postMoveTime,
- tagFamilyMoveStorageToSearchableTags("svc-storage-post", "host-storage-post", "region-storage-post", true),
- tagFamilyMoveStorageToSearchableUpdatedWriteSpec())
- gm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // AwaitRevision is a hint under eventual consistency: this liaison's own cache can
+ // still lag the just-applied move for a moment. A refused write stores nothing, so
+ // retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writePostMoveStatus := writeTagFamilyMoveStream(ctx, clients.StreamWriteClient, groupName, streamName, "stream-storage-post", postMoveTime,
+ tagFamilyMoveStorageToSearchableTags("svc-storage-post", "host-storage-post", "region-storage-post", true),
+ tagFamilyMoveStorageToSearchableUpdatedWriteSpec())
+ innerGm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Verifying the moved stream schema layout")
afterResp, getAfterErr := clients.StreamRegClient.Get(ctx, &databasev1.StreamRegistryServiceGetRequest{
@@ -1153,9 +1184,15 @@ var _ = g.Describe("Schema tag family move", func() {
gm.Expect(clients.AwaitRevision(ctx, max(baseRevision, indexRevision), 10*time.Second)).Should(gm.Succeed())
g.By("Writing and querying measure data before the tag is moved")
- writeStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, preMoveTime,
- tagFamilyMoveTags("measure-svc-pre", "measure-host-pre", "measure-region-pre", false), tagFamilyMoveMeasureInitialWriteSpec(), 1)
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // A single-row write stores nothing when refused, so retrying it after the barrier
+ // above is safe; see the tag-move-in-progress comment below for why a refusal can
+ // still happen here.
+ var writeStatus string
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus = writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, preMoveTime,
+ tagFamilyMoveTags("measure-svc-pre", "measure-host-pre", "measure-region-pre", false), tagFamilyMoveMeasureInitialWriteSpec(), 1)
+ innerGm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
beforeDataPoints := queryTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, begin, end, false, 1, nil)
expectMeasureIndexedMovedTagValue(beforeDataPoints, "measure-svc-pre", "measure-host-pre")
indexedDataPoints := queryTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, begin, end, false, 1, tagFamilyMoveHostCriteria("measure-host-pre"))
@@ -1176,18 +1213,27 @@ var _ = g.Describe("Schema tag family move", func() {
indexMoveRevision := moveTagFamilyMoveMeasureIndexes(ctx, clients, groupName, measureName)
g.By("Continuing measure writes with the old layout while the move is applying")
- writeInMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, inMoveTime,
- tagFamilyMoveTags("measure-svc-in-move", "measure-host-in-move", "measure-region-in-move", false), tagFamilyMoveMeasureInitialWriteSpec(), 2)
- gm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // The entity cache can still lag the move while it is applying; a refused write
+ // stores nothing, so retrying is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeInMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, inMoveTime,
+ tagFamilyMoveTags("measure-svc-in-move", "measure-host-in-move", "measure-region-in-move", false), tagFamilyMoveMeasureInitialWriteSpec(), 2)
+ innerGm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Waiting for the moved measure tag schema and index binding to be applied")
targetRevision := max(updateResp.GetModRevision(), indexMoveRevision)
gm.Expect(clients.AwaitRevision(ctx, targetRevision, 10*time.Second)).Should(gm.Succeed())
g.By("Writing measure data with the new storage-only host layout")
- writePostMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, postMoveTime,
- tagFamilyMoveTags("measure-svc-post", "measure-host-post", "measure-region-post", true), tagFamilyMoveMeasureUpdatedWriteSpec(), 3)
- gm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // AwaitRevision is a hint under eventual consistency: this liaison's own cache can
+ // still lag the just-applied move for a moment. A refused write stores nothing, so
+ // retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writePostMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, postMoveTime,
+ tagFamilyMoveTags("measure-svc-post", "measure-host-post", "measure-region-post", true), tagFamilyMoveMeasureUpdatedWriteSpec(), 3)
+ innerGm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Verifying the moved measure schema layout")
afterResp, getAfterErr := clients.MeasureRegClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{
@@ -1235,10 +1281,16 @@ var _ = g.Describe("Schema tag family move", func() {
gm.Expect(clients.AwaitRevision(ctx, max(baseRevision, indexRevision), 10*time.Second)).Should(gm.Succeed())
g.By("Writing and querying measure data before the storage-only tag is moved")
- writeStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, preMoveTime,
- tagFamilyMoveStorageToSearchableTags("measure-svc-storage-pre", "measure-host-storage-pre", "measure-region-storage-pre", false),
- tagFamilyMoveMeasureStorageToSearchableInitialWriteSpec(), 1)
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // A single-row write stores nothing when refused, so retrying it after the barrier
+ // above is safe; see the tag-move-in-progress comment below for why a refusal can
+ // still happen here.
+ var writeStatus string
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeStatus = writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, preMoveTime,
+ tagFamilyMoveStorageToSearchableTags("measure-svc-storage-pre", "measure-host-storage-pre", "measure-region-storage-pre", false),
+ tagFamilyMoveMeasureStorageToSearchableInitialWriteSpec(), 1)
+ innerGm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
beforeDataPoints := queryStorageToSearchableMoveMeasure(ctx, clients.MeasureWriteClient, groupName, begin, end, false, 1, nil)
expectMeasureStorageOnlyMovedTagValue(beforeDataPoints, "measure-svc-storage-pre", "measure-host-storage-pre")
expectMeasureHostCriteriaError(ctx, clients.MeasureWriteClient, groupName, measureName, "measure-host-storage-pre", begin, end,
@@ -1259,20 +1311,29 @@ var _ = g.Describe("Schema tag family move", func() {
indexMoveRevision := moveTagFamilyMoveMeasureStorageToSearchableIndexes(ctx, clients, groupName, measureName)
g.By("Continuing measure writes with the old storage-only layout while the move is applying")
- writeInMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, inMoveTime,
- tagFamilyMoveStorageToSearchableTags("measure-svc-storage-in-move", "measure-host-storage-in-move", "measure-region-storage-in-move", false),
- tagFamilyMoveMeasureStorageToSearchableInitialWriteSpec(), 2)
- gm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // The entity cache can still lag the move while it is applying; a refused write
+ // stores nothing, so retrying is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writeInMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, inMoveTime,
+ tagFamilyMoveStorageToSearchableTags("measure-svc-storage-in-move", "measure-host-storage-in-move", "measure-region-storage-in-move", false),
+ tagFamilyMoveMeasureStorageToSearchableInitialWriteSpec(), 2)
+ innerGm.Expect(writeInMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Waiting for the moved measure tag schema and index binding to be applied")
targetRevision := max(updateResp.GetModRevision(), indexMoveRevision)
gm.Expect(clients.AwaitRevision(ctx, targetRevision, 10*time.Second)).Should(gm.Succeed())
g.By("Writing measure data with the new searchable host layout")
- writePostMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, postMoveTime,
- tagFamilyMoveStorageToSearchableTags("measure-svc-storage-post", "measure-host-storage-post", "measure-region-storage-post", true),
- tagFamilyMoveMeasureStorageToSearchableUpdatedWriteSpec(), 3)
- gm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ // AwaitRevision is a hint under eventual consistency: this liaison's own cache can
+ // still lag the just-applied move for a moment. A refused write stores nothing, so
+ // retrying it is safe.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ writePostMoveStatus := writeTagFamilyMoveMeasure(ctx, clients.MeasureWriteClient, groupName, measureName, postMoveTime,
+ tagFamilyMoveStorageToSearchableTags("measure-svc-storage-post", "measure-host-storage-post", "measure-region-storage-post", true),
+ tagFamilyMoveMeasureStorageToSearchableUpdatedWriteSpec(), 3)
+ innerGm.Expect(writePostMoveStatus).Should(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
g.By("Verifying the moved measure schema layout")
afterResp, getAfterErr := clients.MeasureRegClient.Get(ctx, &databasev1.MeasureRegistryServiceGetRequest{
diff --git a/test/cases/schema/write_gate.go b/test/cases/schema/write_gate.go
index 7ec637894..88615cff4 100644
--- a/test/cases/schema/write_gate.go
+++ b/test/cases/schema/write_gate.go
@@ -33,6 +33,7 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
)
// wgStreamGroup returns a CATALOG_STREAM Group proto for the write-gate tests.
@@ -313,9 +314,14 @@ var _ = g.Describe("Schema write gate", func() {
gm.Expect(awaitErr).ShouldNot(gm.HaveOccurred())
g.By("Sending a stream write with the old ModRevision R1 (stale)")
- writeStatus, writeErr := sendSingleStreamWrite(ctx, clients.StreamWriteClient, groupName, streamName, r1)
- gm.Expect(writeErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(writeStatus).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
+ // AwaitRevision(r2) confirms only the cluster watermark, not that this liaison's own
+ // entity cache has applied r2; until it has, the cache still shows r1 as current, so a
+ // write naming r1 is indistinguishable from a current write and is accepted
+ // (STATUS_SUCCEED) instead of refused. That stray write stores data in a stream this
+ // spec deletes right after, so nothing later reads it — retrying past it is safe.
+ gm.Eventually(func() (string, error) {
+ return sendSingleStreamWrite(ctx, clients.StreamWriteClient, groupName, streamName, r1)
+ }, flags.EventuallyTimeout).Should(gm.Equal(modelv1.Status_STATUS_EXPIRED_SCHEMA.String()),
"write with client ModRevision < cache must return STATUS_EXPIRED_SCHEMA")
_, _ = clients.GroupClient.Delete(ctx, &databasev1.GroupRegistryServiceDeleteRequest{Group: groupName})
diff --git a/test/cases/stream/data/data.go b/test/cases/stream/data/data.go
index 602ebbfd6..4a5b87ae2 100644
--- a/test/cases/stream/data/data.go
+++ b/test/cases/stream/data/data.go
@@ -68,10 +68,10 @@ var dataFS embed.FS
//go:embed input/*.ql
var qlFS embed.FS
-func expectWriteSucceeded(writeClient streamv1.StreamService_WriteClient) {
- // Setup already waited on SchemaBarrier AwaitSchemaApplied; a NOT_FOUND
- // here means the barrier contract was violated and must fail loudly.
- // Callers must open Write with a deadline so a stalled Recv cannot hang forever.
+// collectWriteStatuses drains a write stream to EOF and returns the status string of every
+// response, in receive order. Callers must open Write with a deadline so a stalled Recv
+// cannot hang forever.
+func collectWriteStatuses(writeClient streamv1.StreamService_WriteClient) []string {
var statuses []string
for {
ack, recvErr := writeClient.Recv()
@@ -81,10 +81,29 @@ func expectWriteSucceeded(writeClient streamv1.StreamService_WriteClient) {
gm.Expect(recvErr).NotTo(gm.HaveOccurred())
statuses = append(statuses, ack.GetStatus())
}
- gm.Expect(statuses).NotTo(gm.BeEmpty())
+ return statuses
+}
+
+// expectWriteSucceeded drains writeClient and asserts every row in this write attempt
+// succeeded. A schema barrier is only a hint under eventual consistency: the liaison's
+// own write cache can still lag it for a moment, so a write that races that lag is
+// refused by status (e.g. STATUS_NOT_FOUND), not a stream error. Call this from inside a
+// gm.Eventually so that refusal is retried; only a refusal that persists past
+// flags.EventuallyTimeout, or a batch that is only partially accepted (gm.StopTrying —
+// resending would duplicate the already-accepted rows), is a real failure.
+func expectWriteSucceeded(innerGm gm.Gomega, writeClient streamv1.StreamService_WriteClient) {
+ statuses := collectWriteStatuses(writeClient)
+ innerGm.Expect(statuses).NotTo(gm.BeEmpty())
+ succeeded := 0
for _, writeStatus := range statuses {
- gm.Expect(writeStatus).To(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < len(statuses) {
+ gm.StopTrying(fmt.Sprintf("stream write partially rejected; statuses=%v", statuses)).Now()
}
+ innerGm.Expect(succeeded).To(gm.Equal(len(statuses)), "stream write rejected; statuses=%v", statuses)
}
// VerifyFn verify whether the query response matches the wanted result.
@@ -323,14 +342,20 @@ func WriteToGroup(conn *grpclib.ClientConn, name, group, fileName string, baseTi
metadata = resp.GetStream().GetMetadata()
c := streamv1.NewStreamServiceClient(conn)
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- elementCounter := 0
- loadData(writeClient, metadata, fmt.Sprintf("%s.json", fileName), baseTime, interval, &elementCounter)
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ // The registry Get above reads the schema server, not this liaison's own write
+ // cache, which can still lag it for a moment; a write that races that lag is
+ // refused by status rather than a stream error. Retry the whole attempt (a fresh
+ // stream, the send, and the drain) together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, writeErr := c.Write(ctx)
+ innerGm.Expect(writeErr).NotTo(gm.HaveOccurred())
+ elementCounter := 0
+ loadData(writeClient, metadata, fmt.Sprintf("%s.json", fileName), baseTime, interval, &elementCounter)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// WriteSpec defines the specification for writing stream data.
@@ -342,43 +367,49 @@ type WriteSpec struct {
// WriteMixed writes stream data in schema order first, and then in spec order.
func WriteMixed(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration, writeSpecs ...WriteSpec) {
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
schemaClient := databasev1.NewStreamRegistryServiceClient(conn)
c := streamv1.NewStreamServiceClient(conn)
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- var currentMd *commonv1.Metadata
- elementCounter := 0
- currentTime := baseTime
- for idx, ws := range writeSpecs {
- if ws.Metadata != nil {
- resp, getErr := schemaClient.Get(ctx, &databasev1.StreamRegistryServiceGetRequest{Metadata: ws.Metadata})
- gm.Expect(getErr).NotTo(gm.HaveOccurred())
- currentMd = resp.GetStream().GetMetadata()
- }
- gm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
-
- var templates []interface{}
- content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
- gm.Expect(readErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
-
- if idx == 0 {
- loadData(writeClient, currentMd, ws.DataFile, currentTime, interval, &elementCounter)
- } else {
- var mdToSend *commonv1.Metadata
+ // The liaison's write cache can still lag a schema barrier, refusing a write by
+ // status rather than a stream error, so the whole attempt (a fresh stream, every
+ // schema Get and loadData/loadDataWithSpec call, and the drain) is retried together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, err := c.Write(ctx)
+ innerGm.Expect(err).NotTo(gm.HaveOccurred())
+
+ var currentMd *commonv1.Metadata
+ elementCounter := 0
+ currentTime := baseTime
+ for idx, ws := range writeSpecs {
if ws.Metadata != nil {
- mdToSend = currentMd
+ resp, getErr := schemaClient.Get(ctx, &databasev1.StreamRegistryServiceGetRequest{Metadata: ws.Metadata})
+ innerGm.Expect(getErr).NotTo(gm.HaveOccurred())
+ currentMd = resp.GetStream().GetMetadata()
+ }
+ innerGm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
+
+ var templates []interface{}
+ content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
+ innerGm.Expect(readErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
+
+ if idx == 0 {
+ loadData(writeClient, currentMd, ws.DataFile, currentTime, interval, &elementCounter)
+ } else {
+ var mdToSend *commonv1.Metadata
+ if ws.Metadata != nil {
+ mdToSend = currentMd
+ }
+ loadDataWithSpec(writeClient, mdToSend, ws.DataFile, currentTime, interval, ws.Spec, &elementCounter)
}
- loadDataWithSpec(writeClient, mdToSend, ws.DataFile, currentTime, interval, ws.Spec, &elementCounter)
+ currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
}
- currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
- }
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// loadDataWithElementIDMap loads data with element IDs specified in the data file for deduplication tests.
@@ -488,11 +519,17 @@ func WriteDeduplicationTest(conn *grpclib.ClientConn, name string, baseTime time
metadata = resp.GetStream().GetMetadata()
c := streamv1.NewStreamServiceClient(conn)
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- loadDataWithElementIDMap(writeClient, metadata, fmt.Sprintf("%s.json", name), baseTime, interval)
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ // The registry Get above reads the schema server, not this liaison's own write
+ // cache, which can still lag it for a moment; a write that races that lag is
+ // refused by status rather than a stream error. Retry the whole attempt (a fresh
+ // stream, the send, and the drain) together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, writeErr := c.Write(ctx)
+ innerGm.Expect(writeErr).NotTo(gm.HaveOccurred())
+ loadDataWithElementIDMap(writeClient, metadata, fmt.Sprintf("%s.json", name), baseTime, interval)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
diff --git a/test/cases/trace/data/data.go b/test/cases/trace/data/data.go
index 0429bbb7f..9f33f62c3 100644
--- a/test/cases/trace/data/data.go
+++ b/test/cases/trace/data/data.go
@@ -66,10 +66,10 @@ var qlFS embed.FS
//go:embed testdata/*.json
var dataFS embed.FS
-func expectWriteSucceeded(writeClient tracev1.TraceService_WriteClient) {
- // Setup already waited on SchemaBarrier AwaitSchemaApplied; a NOT_FOUND
- // here means the barrier contract was violated and must fail loudly.
- // Callers must open Write with a deadline so a stalled Recv cannot hang forever.
+// collectWriteStatuses drains a write stream to EOF and returns the status string of every
+// response, in receive order. Callers must open Write with a deadline so a stalled Recv
+// cannot hang forever.
+func collectWriteStatuses(writeClient tracev1.TraceService_WriteClient) []string {
var statuses []string
for {
ack, recvErr := writeClient.Recv()
@@ -79,10 +79,29 @@ func expectWriteSucceeded(writeClient tracev1.TraceService_WriteClient) {
gm.Expect(recvErr).NotTo(gm.HaveOccurred())
statuses = append(statuses, ack.GetStatus())
}
- gm.Expect(statuses).NotTo(gm.BeEmpty())
+ return statuses
+}
+
+// expectWriteSucceeded drains writeClient and asserts every row in this write attempt
+// succeeded. A schema barrier is only a hint under eventual consistency: the liaison's
+// own write cache can still lag it for a moment, so a write that races that lag is
+// refused by status (e.g. STATUS_NOT_FOUND), not a stream error. Call this from inside a
+// gm.Eventually so that refusal is retried; only a refusal that persists past
+// flags.EventuallyTimeout, or a batch that is only partially accepted (gm.StopTrying —
+// resending would duplicate the already-accepted rows), is a real failure.
+func expectWriteSucceeded(innerGm gm.Gomega, writeClient tracev1.TraceService_WriteClient) {
+ statuses := collectWriteStatuses(writeClient)
+ innerGm.Expect(statuses).NotTo(gm.BeEmpty())
+ succeeded := 0
for _, writeStatus := range statuses {
- gm.Expect(writeStatus).To(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()))
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
}
+ if succeeded > 0 && succeeded < len(statuses) {
+ gm.StopTrying(fmt.Sprintf("trace write partially rejected; statuses=%v", statuses)).Now()
+ }
+ innerGm.Expect(succeeded).To(gm.Equal(len(statuses)), "trace write rejected; statuses=%v", statuses)
}
// VerifyFn verify whether the query response matches the wanted result.
@@ -394,20 +413,28 @@ func WriteToGroup(conn *grpclib.ClientConn, name, group, fileName string, baseTi
}
schema := databasev1.NewTraceRegistryServiceClient(conn)
resp, err := schema.Get(context.Background(), &databasev1.TraceRegistryServiceGetRequest{Metadata: metadata})
- if err != nil {
- return
- }
+ // Get reads the schema server, not a liaison's write cache, so a failure here is
+ // not the liaison-cache-lag race this function's write retries guard against — it
+ // more likely means a genuine test bug (wrong name/group, or called before the
+ // schema was created), so fail loudly instead of silently no-oping the write.
+ gm.Expect(err).NotTo(gm.HaveOccurred())
metadata = resp.GetTrace().GetMetadata()
c := tracev1.NewTraceServiceClient(conn)
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
- version := uint64(1)
- loadData(writeClient, metadata, fmt.Sprintf("%s.json", fileName), baseTime, interval, &version)
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ // The registry Get above reads the schema server, not this liaison's own write
+ // cache, which can still lag it for a moment; a write that races that lag is
+ // refused by status rather than a stream error. Retry the whole attempt (a fresh
+ // stream, the send, and the drain) together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, writeErr := c.Write(ctx)
+ innerGm.Expect(writeErr).NotTo(gm.HaveOccurred())
+ version := uint64(1)
+ loadData(writeClient, metadata, fmt.Sprintf("%s.json", fileName), baseTime, interval, &version)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// WriteSpec defines the specification for writing trace data.
@@ -419,43 +446,49 @@ type WriteSpec struct {
// WriteMixed writes trace data in schema order first, and then in spec order.
func WriteMixed(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration, writeSpecs ...WriteSpec) {
- ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
- defer cancel()
schemaClient := databasev1.NewTraceRegistryServiceClient(conn)
c := tracev1.NewTraceServiceClient(conn)
- writeClient, err := c.Write(ctx)
- gm.Expect(err).NotTo(gm.HaveOccurred())
-
- var currentMd *commonv1.Metadata
- version := uint64(1)
- currentTime := baseTime
- for idx, ws := range writeSpecs {
- if ws.Metadata != nil {
- resp, getErr := schemaClient.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{Metadata: ws.Metadata})
- gm.Expect(getErr).NotTo(gm.HaveOccurred())
- currentMd = resp.GetTrace().GetMetadata()
- }
- gm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
- var templates []interface{}
- content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
- gm.Expect(readErr).ShouldNot(gm.HaveOccurred())
- gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
-
- if idx == 0 {
- loadData(writeClient, currentMd, ws.DataFile, currentTime, interval, &version)
- } else {
- var mdToSend *commonv1.Metadata
+ // The liaison's write cache can still lag a schema barrier, refusing a write by
+ // status rather than a stream error, so the whole attempt (a fresh stream, every
+ // schema Get and loadData/loadDataWithSpec call, and the drain) is retried together.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
+ defer cancel()
+ writeClient, err := c.Write(ctx)
+ innerGm.Expect(err).NotTo(gm.HaveOccurred())
+
+ var currentMd *commonv1.Metadata
+ version := uint64(1)
+ currentTime := baseTime
+ for idx, ws := range writeSpecs {
if ws.Metadata != nil {
- mdToSend = currentMd
+ resp, getErr := schemaClient.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{Metadata: ws.Metadata})
+ innerGm.Expect(getErr).NotTo(gm.HaveOccurred())
+ currentMd = resp.GetTrace().GetMetadata()
+ }
+ innerGm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")
+
+ var templates []interface{}
+ content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
+ innerGm.Expect(readErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
+
+ if idx == 0 {
+ loadData(writeClient, currentMd, ws.DataFile, currentTime, interval, &version)
+ } else {
+ var mdToSend *commonv1.Metadata
+ if ws.Metadata != nil {
+ mdToSend = currentMd
+ }
+ loadDataWithSpec(writeClient, mdToSend, ws.DataFile, currentTime, interval, ws.Spec, &version)
}
- loadDataWithSpec(writeClient, mdToSend, ws.DataFile, currentTime, interval, ws.Spec, &version)
+ currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
}
- currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
- }
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- expectWriteSucceeded(writeClient)
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
+ expectWriteSucceeded(innerGm, writeClient)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// unmarshalYAMLWithSpanEncoding decodes YAML with special handling for span data.
diff --git a/test/cases/tracepipeline/data/data.go b/test/cases/tracepipeline/data/data.go
index 864aee9f8..2f0b6c4a7 100644
--- a/test/cases/tracepipeline/data/data.go
+++ b/test/cases/tracepipeline/data/data.go
@@ -25,6 +25,7 @@ import (
"encoding/base64"
"encoding/json"
"errors"
+ "fmt"
"io"
"regexp"
"slices"
@@ -272,12 +273,38 @@ func WriteBatchEntries(conn *grpclib.ClientConn, group string, baseTime time.Tim
gm.Expect(schemaErr).NotTo(gm.HaveOccurred())
md = resp.GetTrace().GetMetadata()
+ // The liaison's write cache can lag the schema registry under eventual
+ // consistency, and it refuses a write by status (not by stream error) until it
+ // catches up. Retry while every row is refused: a refused row stores nothing.
+ // A partial success cannot be retried without duplicating the accepted rows,
+ // so it fails at once.
+ gm.Eventually(func(innerGm gm.Gomega) {
+ statuses := sendTraceRows(innerGm, conn, md, baseTime, interval, rows)
+ innerGm.Expect(statuses).To(gm.HaveLen(len(rows)), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < len(rows) {
+ gm.StopTrying(fmt.Sprintf("trace write to %s/%s partially rejected; statuses=%v", group, md.GetName(), statuses)).Now()
+ }
+ innerGm.Expect(succeeded).To(gm.Equal(len(rows)),
+ "trace write to %s/%s rejected; statuses=%v", group, md.GetName(), statuses)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
+}
+
+// sendTraceRows writes rows on one write stream and returns the status of every response.
+func sendTraceRows(innerGm gm.Gomega, conn *grpclib.ClientConn, md *commonv1.Metadata,
+ baseTime time.Time, interval time.Duration, rows []TraceRow,
+) []string {
c := tracev1.NewTraceServiceClient(conn)
// The deadline bounds the response drain below, which blocks in Recv.
writeCtx, writeCancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
defer writeCancel()
writeClient, writeErr := c.Write(writeCtx)
- gm.Expect(writeErr).NotTo(gm.HaveOccurred())
+ innerGm.Expect(writeErr).NotTo(gm.HaveOccurred())
for version, row := range rows {
// Compute the timestamp for this row and build the full tag slice with
@@ -299,27 +326,20 @@ func WriteBatchEntries(conn *grpclib.ClientConn, group string, baseTime time.Tim
Span: []byte(row.Span),
Version: uint64(version + 1),
})
- gm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
}
- gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
- // The liaison rejects a write by status, not by stream error, so a row
- // it refused (schema not found or not applied, invalid timestamp, ...)
- // would otherwise vanish and surface only as an empty query much later.
+ innerGm.Expect(writeClient.CloseSend()).To(gm.Succeed())
var statuses []string
for {
resp, recvErr := writeClient.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
- gm.Expect(recvErr).NotTo(gm.HaveOccurred())
+ innerGm.Expect(recvErr).NotTo(gm.HaveOccurred())
statuses = append(statuses, resp.GetStatus())
}
- gm.Expect(statuses).To(gm.HaveLen(len(rows)), "one write response per row")
- for _, writeStatus := range statuses {
- gm.Expect(writeStatus).To(gm.Equal(modelv1.Status_STATUS_SUCCEED.String()),
- "trace write to %s/%s rejected; statuses=%v", group, md.GetName(), statuses)
- }
+ return statuses
}
// unmarshalYAMLWithSpanEncoding decodes YAML with special handling for span data.
diff --git a/test/e2e/tracepipeline/e2e_test.go b/test/e2e/tracepipeline/e2e_test.go
index 835115209..b262b4512 100644
--- a/test/e2e/tracepipeline/e2e_test.go
+++ b/test/e2e/tracepipeline/e2e_test.go
@@ -47,6 +47,7 @@ import (
"net"
"net/http"
"os"
+ "slices"
"strconv"
"strings"
"testing"
@@ -67,6 +68,7 @@ import (
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema/property"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
tracepipeline "github.com/apache/skywalking-banyandb/test/cases/tracepipeline"
tracepipelinedata "github.com/apache/skywalking-banyandb/test/cases/tracepipeline/data"
)
@@ -208,34 +210,61 @@ func waitForTraceSchema() {
gomega.Expect(err).NotTo(gomega.HaveOccurred())
defer func() { _ = conn.Close() }()
- c := databasev1.NewTraceRegistryServiceClient(conn)
- var revision int64
+ // Collect the highest mod_revision across everything PreloadSchema creates
+ // (group, trace, index rules, index rule bindings): the writes and the
+ // duration-ordered queries depend on all of them, not on the trace alone.
+ groupClient := databasev1.NewGroupRegistryServiceClient(conn)
+ traceClient := databasev1.NewTraceRegistryServiceClient(conn)
+ indexRuleClient := databasev1.NewIndexRuleRegistryServiceClient(conn)
+ bindingClient := databasev1.NewIndexRuleBindingRegistryServiceClient(conn)
+ var maxRevision int64
gomega.Eventually(func(innerGm gomega.Gomega) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
- resp, getErr := c.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{
- Metadata: &commonv1.Metadata{Name: "filter", Group: tracepipeline.PipelineGroup},
+ group := tracepipeline.PipelineGroup
+ groupResp, groupErr := groupClient.Get(ctx, &databasev1.GroupRegistryServiceGetRequest{Group: group})
+ innerGm.Expect(groupErr).NotTo(gomega.HaveOccurred(), "group %s not yet visible on the standalone", group)
+ traceResp, traceErr := traceClient.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{
+ Metadata: &commonv1.Metadata{Name: "filter", Group: group},
})
- innerGm.Expect(getErr).NotTo(gomega.HaveOccurred(),
- "trace schema filter/%s not yet visible on the standalone", tracepipeline.PipelineGroup)
- revision = resp.GetTrace().GetMetadata().GetModRevision()
- }, 60*time.Second, 500*time.Millisecond).Should(gomega.Succeed())
+ innerGm.Expect(traceErr).NotTo(gomega.HaveOccurred(), "trace schema filter/%s not yet visible on the standalone", group)
+ ruleResp, ruleErr := indexRuleClient.List(ctx, &databasev1.IndexRuleRegistryServiceListRequest{Group: group})
+ innerGm.Expect(ruleErr).NotTo(gomega.HaveOccurred())
+ innerGm.Expect(ruleResp.GetIndexRule()).NotTo(gomega.BeEmpty(), "index rules of %s not yet visible", group)
+ bindingResp, bindingErr := bindingClient.List(ctx, &databasev1.IndexRuleBindingRegistryServiceListRequest{Group: group})
+ innerGm.Expect(bindingErr).NotTo(gomega.HaveOccurred())
+ innerGm.Expect(bindingResp.GetIndexRuleBinding()).NotTo(gomega.BeEmpty(), "index rule bindings of %s not yet visible", group)
+
+ revisions := []int64{
+ groupResp.GetGroup().GetMetadata().GetModRevision(),
+ traceResp.GetTrace().GetMetadata().GetModRevision(),
+ }
+ for _, rule := range ruleResp.GetIndexRule() {
+ revisions = append(revisions, rule.GetMetadata().GetModRevision())
+ }
+ for _, binding := range bindingResp.GetIndexRuleBinding() {
+ revisions = append(revisions, binding.GetMetadata().GetModRevision())
+ }
+ maxRevision = slices.Max(revisions)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
// The registry answering is not enough: writes validate against the
- // liaison's local schema cache, which can lag the property schema store
- // and refuses a write with STATUS_NOT_FOUND until it catches up. Barrier
- // on that cache, as pkg/test/setup does for the integration suites.
- const barrierTimeout = 60 * time.Second
- ctx, cancel := context.WithTimeout(context.Background(), barrierTimeout)
+ // liaison's local schema cache, which can lag the property schema store.
+ // The standalone applies schema events on one watch session in order, and
+ // PreloadSchema creates them sequentially, so the revision watermark
+ // reaching the highest preloaded revision means every preloaded schema's
+ // handlers have run. Under eventual consistency this is still a hint, not
+ // a guarantee; the writes retry on a refused status.
+ ctx, cancel := context.WithTimeout(context.Background(), flags.EventuallyTimeout)
defer cancel()
- resp, awaitErr := schemav1.NewSchemaBarrierServiceClient(conn).AwaitSchemaApplied(ctx, &schemav1.AwaitSchemaAppliedRequest{
- Keys: []*schemav1.SchemaKey{{Kind: "trace", Group: tracepipeline.PipelineGroup, Name: "filter"}},
- MinRevisions: []int64{revision},
- Timeout: durationpb.New(barrierTimeout),
+ resp, awaitErr := schemav1.NewSchemaBarrierServiceClient(conn).AwaitRevisionApplied(ctx, &schemav1.AwaitRevisionAppliedRequest{
+ MinRevision: maxRevision,
+ Timeout: durationpb.New(flags.EventuallyTimeout),
})
gomega.Expect(awaitErr).NotTo(gomega.HaveOccurred())
gomega.Expect(resp.GetApplied()).To(gomega.BeTrue(),
- "trace schema filter/%s not applied to local caches; laggards=%v", tracepipeline.PipelineGroup, resp.GetLaggards())
+ "preloaded schemas of %s (max revision %d) not applied to local caches; laggards=%v",
+ tracepipeline.PipelineGroup, maxRevision, resp.GetLaggards())
}
// waitForServerReachable polls the gRPC endpoint until the standalone
diff --git a/test/integration/distributed/backup/common.go b/test/integration/distributed/backup/common.go
index 1ae1897fd..ce2c1a4e6 100644
--- a/test/integration/distributed/backup/common.go
+++ b/test/integration/distributed/backup/common.go
@@ -129,15 +129,22 @@ func InitializeTestSuite() (*CommonTestVars, error) {
Name: "p",
Group: "g",
}
- resp, err := client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
- Metadata: md,
- Id: "1",
- Tags: []*modelv1.Tag{
- {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
- },
- }})
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ // A just-created group and property reach the liaison's in-memory repositories
+ // asynchronously via the schema watch, so the first Apply right after creating them
+ // can transiently fail. Eventually rides out that propagation window.
+ var resp *propertyv1.ApplyResponse
+ gomega.Eventually(func() error {
+ var applyErr error
+ resp, applyErr = client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
+ Metadata: md,
+ Id: "1",
+ Tags: []*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
+ },
+ }})
+ return applyErr
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
gomega.Expect(resp.Created).To(gomega.BeTrue())
gomega.Expect(resp.TagsNum).To(gomega.Equal(uint32(2)))
diff --git a/test/integration/distributed/deletion/common.go b/test/integration/distributed/deletion/common.go
index 9da1aa7c1..2b112207d 100644
--- a/test/integration/distributed/deletion/common.go
+++ b/test/integration/distributed/deletion/common.go
@@ -35,12 +35,14 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ schemav1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/schema/v1"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/pool"
@@ -61,15 +63,16 @@ type SetupResult struct {
var SetupFunc func() SetupResult
var (
- result SetupResult
- connection *grpc.ClientConn
- groupClient databasev1.GroupRegistryServiceClient
- measureRegClient databasev1.MeasureRegistryServiceClient
- measureWriteClient measurev1.MeasureServiceClient
- dataNode0Path string
- dataNode1Path string
- liaisonNodePath string
- goods []gleak.Goroutine
+ result SetupResult
+ connection *grpc.ClientConn
+ groupClient databasev1.GroupRegistryServiceClient
+ measureRegClient databasev1.MeasureRegistryServiceClient
+ measureWriteClient measurev1.MeasureServiceClient
+ measureBarrierClient schemav1.SchemaBarrierServiceClient
+ dataNode0Path string
+ dataNode1Path string
+ liaisonNodePath string
+ goods []gleak.Goroutine
)
var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
@@ -96,6 +99,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
groupClient = databasev1.NewGroupRegistryServiceClient(connection)
measureRegClient = databasev1.NewMeasureRegistryServiceClient(connection)
measureWriteClient = measurev1.NewMeasureServiceClient(connection)
+ measureBarrierClient = schemav1.NewSchemaBarrierServiceClient(connection)
})
var _ = ginkgo.SynchronizedAfterSuite(func() {
@@ -376,8 +380,16 @@ var _ = ginkgo.Describe("GroupDeletion", func() {
})
})
+// createMeasureInGroup creates measure in group and waits for the cluster's schema
+// watermark to reach the Create response's mod_revision before returning.
+//
+// AwaitRevisionApplied is a hint under eventual consistency: it confirms the cluster
+// watermark, not that this liaison's own entity cache has applied the schema yet.
+// writeMeasureDataToGroup retries a refused write to tolerate that residual per-liaison
+// lag, so the barrier only needs to replace the blind sleep this used to rely on for the
+// common case.
func createMeasureInGroup(regClient databasev1.MeasureRegistryServiceClient, group, measure string) {
- _, createErr := regClient.Create(context.TODO(), &databasev1.MeasureRegistryServiceCreateRequest{
+ createResp, createErr := regClient.Create(context.TODO(), &databasev1.MeasureRegistryServiceCreateRequest{
Measure: &databasev1.Measure{
Metadata: &commonv1.Metadata{Name: measure, Group: group},
Entity: &databasev1.Entity{TagNames: []string{"id"}},
@@ -394,16 +406,25 @@ func createMeasureInGroup(regClient databasev1.MeasureRegistryServiceClient, gro
},
})
gomega.Expect(createErr).ShouldNot(gomega.HaveOccurred())
- time.Sleep(2 * time.Second)
+ barrierResp, barrierErr := measureBarrierClient.AwaitRevisionApplied(context.TODO(), &schemav1.AwaitRevisionAppliedRequest{
+ MinRevision: createResp.GetModRevision(),
+ Timeout: durationpb.New(flags.EventuallyTimeout),
+ })
+ gomega.Expect(barrierErr).ShouldNot(gomega.HaveOccurred())
+ gomega.Expect(barrierResp.GetApplied()).Should(gomega.BeTrue(), "timeout waiting for schema revision %d", createResp.GetModRevision())
}
-func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) {
+// sendMeasureBatchToGroup writes count distinct measure rows on one write stream and returns
+// the status string of every response, in send order.
+func sendMeasureBatchToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) ([]string, error) {
wc, wcErr := writeClient.Write(context.TODO())
- gomega.Expect(wcErr).ShouldNot(gomega.HaveOccurred())
+ if wcErr != nil {
+ return nil, wcErr
+ }
metadata := &commonv1.Metadata{Name: measure, Group: group}
baseTime := time.Now().Truncate(time.Millisecond)
for idx := 0; idx < count; idx++ {
- sendErr := wc.Send(&measurev1.WriteRequest{
+ if sendErr := wc.Send(&measurev1.WriteRequest{
Metadata: metadata,
DataPoint: &measurev1.DataPointValue{
Timestamp: timestamppb.New(baseTime.Add(time.Duration(idx) * time.Second)),
@@ -417,16 +438,44 @@ func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group,
}},
},
MessageId: uint64(time.Now().UnixNano() + int64(idx)),
- })
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ }); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeSendErr := wc.CloseSend(); closeSendErr != nil {
+ return nil, closeSendErr
}
- closeSendErr := wc.CloseSend()
- gomega.Expect(closeSendErr).ShouldNot(gomega.HaveOccurred())
+ var statuses []string
for {
- _, recvErr := wc.Recv()
+ resp, recvErr := wc.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
- gomega.Expect(recvErr).ShouldNot(gomega.HaveOccurred())
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
}
+ return statuses, nil
+}
+
+// writeMeasureDataToGroup retries sendMeasureBatchToGroup until every row is accepted. Rows can be
+// refused by status until the liaison's schema cache catches up. A measure row is keyed by
+// series and timestamp with the latest version winning, so resending the batch with the same
+// values is idempotent: any refusal, including a partly accepted batch, is retried within
+// flags.EventuallyTimeout.
+func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) {
+ gomega.Eventually(func(innerGomega gomega.Gomega) {
+ statuses, sendErr := sendMeasureBatchToGroup(writeClient, group, measure, count)
+ innerGomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ innerGomega.Expect(statuses).Should(gomega.HaveLen(count), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ innerGomega.Expect(succeeded).Should(gomega.Equal(count),
+ "measure write to %s/%s rejected; statuses=%v", group, measure, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
}
diff --git a/test/integration/distributed/inspection/common.go b/test/integration/distributed/inspection/common.go
index a75f99b46..e2cddfd2f 100644
--- a/test/integration/distributed/inspection/common.go
+++ b/test/integration/distributed/inspection/common.go
@@ -20,6 +20,7 @@ package inspection
import (
"context"
+ "errors"
"fmt"
"io"
"strconv"
@@ -115,10 +116,13 @@ var _ = ginkgo.ReportAfterSuite("Distributed Inspect Suite", func(report ginkgo.
}
})
-func writeMeasureData(ctx context.Context, groupName, measureName string, dataCount int) {
+// sendMeasureDataBatch writes dataCount distinct measure rows on one write stream and
+// returns the status string of every response, in send order.
+func sendMeasureDataBatch(ctx context.Context, groupName, measureName string, dataCount int) ([]string, error) {
writeClient, writeErr := measureWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
-
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: measureName,
Group: groupName,
@@ -140,20 +144,55 @@ func writeMeasureData(ctx context.Context, groupName, measureName string, dataCo
},
MessageId: uint64(time.Now().UnixNano()),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
+ }
+ return statuses, nil
}
-func writeStreamData(ctx context.Context, groupName, streamName string, dataCount int) {
- writeClient, writeErr := streamWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
+// writeMeasureData retries sendMeasureDataBatch until every row is accepted. Rows can be
+// refused by status until the liaison's schema cache catches up. A measure row is keyed by
+// series and timestamp with the latest version winning, so resending the batch with the same
+// values is idempotent: any refusal, including a partly accepted batch, is retried within
+// flags.EventuallyTimeout.
+func writeMeasureData(ctx context.Context, groupName, measureName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendMeasureDataBatch(ctx, groupName, measureName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "measure write to %s/%s rejected; statuses=%v", groupName, measureName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+}
+// sendStreamDataBatch writes dataCount distinct stream rows on one write stream and returns
+// the status string of every response, in send order.
+func sendStreamDataBatch(ctx context.Context, groupName, streamName string, dataCount int) ([]string, error) {
+ writeClient, writeErr := streamWriteClient.Write(ctx)
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: streamName,
Group: groupName,
@@ -173,20 +212,58 @@ func writeStreamData(ctx context.Context, groupName, streamName string, dataCoun
},
MessageId: uint64(time.Now().UnixNano()),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
+ }
+ return statuses, nil
}
-func writeTraceData(ctx context.Context, groupName, traceName string, dataCount int) {
- writeClient, writeErr := traceWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
+// writeStreamData retries sendStreamDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag behind the group/schema it was just given, and a
+// refused row stores nothing, so repeating the whole batch is safe. If some rows succeed and
+// some are refused, the batch cannot be safely repeated without duplicating the accepted
+// rows, so it stops retrying at once.
+func writeStreamData(ctx context.Context, groupName, streamName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendStreamDataBatch(ctx, groupName, streamName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < dataCount {
+ gomega.StopTrying(fmt.Sprintf("stream write to %s/%s partially rejected; statuses=%v", groupName, streamName, statuses)).Now()
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "stream write to %s/%s rejected; statuses=%v", groupName, streamName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+}
+// sendTraceDataBatch writes dataCount distinct trace rows on one write stream and returns the
+// status string of every response, in send order.
+func sendTraceDataBatch(ctx context.Context, groupName, traceName string, dataCount int) ([]string, error) {
+ writeClient, writeErr := traceWriteClient.Write(ctx)
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: traceName,
Group: groupName,
@@ -207,14 +284,49 @@ func writeTraceData(ctx context.Context, groupName, traceName string, dataCount
Span: []byte(fmt.Sprintf("span_data_%d", idx)),
Version: uint64(idx + 1),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
+ }
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ return statuses, nil
+}
+
+// writeTraceData retries sendTraceDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag behind the group/schema it was just given, and a
+// refused row stores nothing, so repeating the whole batch is safe. If some rows succeed and
+// some are refused, the batch cannot be safely repeated without duplicating the accepted
+// rows, so it stops retrying at once.
+func writeTraceData(ctx context.Context, groupName, traceName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendTraceDataBatch(ctx, groupName, traceName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < dataCount {
+ gomega.StopTrying(fmt.Sprintf("trace write to %s/%s partially rejected; statuses=%v", groupName, traceName, statuses)).Now()
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "trace write to %s/%s rejected; statuses=%v", groupName, traceName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
}
var _ = ginkgo.Describe("Inspect in distributed mode", func() {
@@ -305,24 +417,28 @@ var _ = ginkgo.Describe("Inspect in distributed mode", func() {
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
-
- ginkgo.By("Verifying data collected from multiple nodes")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one data node")
-
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total data size across all nodes: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // measure may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+
+ ginkgo.By("Verifying data collected from multiple nodes")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one data node")
+
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total data size across all nodes: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return liaison info", func() {
@@ -424,24 +540,28 @@ var _ = ginkgo.Describe("Inspect stream in distributed mode", func() {
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting stream group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
-
- ginkgo.By("Verifying stream data collected from multiple nodes")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one stream data node")
-
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "stream data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "stream data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Stream data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total stream data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total stream data size: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // stream may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting stream group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+
+ ginkgo.By("Verifying stream data collected from multiple nodes")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one stream data node")
+
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "stream data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "stream data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Stream data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total stream data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total stream data size: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return liaison info", func() {
@@ -544,24 +664,28 @@ var _ = ginkgo.Describe("Inspect trace in distributed mode", func() {
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting trace group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
-
- ginkgo.By("Verifying trace data collected from multiple nodes")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one trace data node")
-
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "trace data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "trace data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Trace data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total trace data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total trace data size: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // trace may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting trace group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+
+ ginkgo.By("Verifying trace data collected from multiple nodes")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from at least one trace data node")
+
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "trace data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "trace data node %d (%s) should have DataSizeBytes > 0", idx, dataInfo.Node.Metadata.Name)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Trace data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total trace data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total trace data size: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return liaison info", func() {
diff --git a/test/integration/distributed/rbac/rbac_test.go b/test/integration/distributed/rbac/rbac_test.go
index f8bcf6946..6d46a285f 100644
--- a/test/integration/distributed/rbac/rbac_test.go
+++ b/test/integration/distributed/rbac/rbac_test.go
@@ -151,37 +151,68 @@ func distributedMarkers(response *measurev1.QueryResponse) []string {
return markers
}
-func sendDistributedFrames(
+// sendDistributedFrameStatuses opens a real bidirectional write through actor, sends every
+// frame, closes the send side, and returns the status string of each response in send order.
+// A transport-level error (e.g. codes.PermissionDenied for a refused stream) ends the stream
+// before any status is returned.
+func sendDistributedFrameStatuses(
actor distributedActor,
client measurev1.MeasureServiceClient,
frames ...*measurev1.WriteRequest,
-) error {
+) ([]string, error) {
stream, openErr := client.Write(actor.context())
if openErr != nil {
- return openErr
+ return nil, openErr
}
for _, frame := range frames {
if sendErr := stream.Send(frame); sendErr != nil {
if errors.Is(sendErr, io.EOF) {
break
}
- return sendErr
+ return nil, sendErr
}
}
if closeErr := stream.CloseSend(); closeErr != nil {
- return closeErr
+ return nil, closeErr
}
+ var statuses []string
for {
- _, receiveErr := stream.Recv()
+ resp, receiveErr := stream.Recv()
if errors.Is(receiveErr, io.EOF) {
- return nil
+ return statuses, nil
}
if receiveErr != nil {
- return receiveErr
+ return statuses, receiveErr
}
+ statuses = append(statuses, resp.GetStatus())
}
}
+// seedDistributedFrames sends frames through actor and retries until every frame is accepted. The schema barrier awaited before this call only confirms liaison B's cache
+// (AwaitSchemaApplied is issued against barrierB); liaison A, which this seed writes through,
+// can still lag the same schema for a moment and refuse a frame by status rather than a stream
+// error. A refused frame stores nothing, and a measure row is keyed by series and timestamp with
+// the latest version winning, so resending the whole batch, even a partly accepted one, is
+// idempotent.
+func seedDistributedFrames(
+ actor distributedActor,
+ client measurev1.MeasureServiceClient,
+ frames ...*measurev1.WriteRequest,
+) {
+ gm.EventuallyWithOffset(1, func(innerGm gm.Gomega) {
+ statuses, sendErr := sendDistributedFrameStatuses(actor, client, frames...)
+ innerGm.Expect(sendErr).NotTo(gm.HaveOccurred())
+ innerGm.Expect(statuses).To(gm.HaveLen(len(frames)), "one write response per frame")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ innerGm.Expect(succeeded).To(gm.Equal(len(frames)), "seed write by %s rejected; statuses=%v", actor.username, statuses)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
+}
+
var _ = g.Describe("rbac-data distributed public-path smoke", func() {
var (
closeCluster func()
@@ -263,10 +294,10 @@ var _ = g.Describe("rbac-data distributed public-path smoke", func() {
})
gm.Expect(barrierErr).NotTo(gm.HaveOccurred(), "the distributed fixture schema must converge before public data calls")
- gm.Expect(sendDistributedFrames(distributedAdmin, measuresA,
+ seedDistributedFrames(distributedAdmin, measuresA,
distributedMarkerFrame(distributedAlpha, alphaMarker, baseTime, 1),
distributedMarkerFrame(distributedBeta, betaMarker, baseTime, 2),
- )).To(gm.Succeed(), "the administrator must seed both groups through liaison A")
+ )
gm.Eventually(func() []string {
response, queryErr := measuresB.Query(distributedAdmin.context(), distributedMarkerQuery(baseTime, distributedAlpha, distributedBeta))
diff --git a/test/integration/handoff/handoff_suite_test.go b/test/integration/handoff/handoff_suite_test.go
index 0a097cb3d..03cb585df 100644
--- a/test/integration/handoff/handoff_suite_test.go
+++ b/test/integration/handoff/handoff_suite_test.go
@@ -23,7 +23,6 @@ import (
"errors"
"fmt"
"hash/fnv"
- "io"
"os"
"path/filepath"
"sort"
@@ -397,12 +396,6 @@ var _ = Describe("trace handoff", func() {
func writeTrace(conn *grpc.ClientConn, traceID string) time.Time {
client := tracev1.NewTraceServiceClient(conn)
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- stream, err := client.Write(ctx)
- Expect(err).NotTo(HaveOccurred())
-
baseTime := time.Now().UTC().Truncate(time.Millisecond)
tags := []*modelv1.TagValue{
@@ -416,19 +409,43 @@ func writeTrace(conn *grpc.ClientConn, traceID string) time.Time {
{Value: &modelv1.TagValue_Timestamp{Timestamp: timestamppb.New(baseTime)}},
}
- err = stream.Send(&tracev1.WriteRequest{
- Metadata: &commonv1.Metadata{Name: "sw", Group: "test-trace-group"},
- Tags: tags,
- Span: []byte("span-" + traceID),
- Version: 1,
- })
- Expect(err).NotTo(HaveOccurred())
- Expect(stream.CloseSend()).To(Succeed())
-
- Eventually(func() error {
- _, err = stream.Recv()
- return err
- }, flags.EventuallyTimeout).Should(Equal(io.EOF))
+ // This suite's harness hand-rolls data-node/liaison bring-up (newDataNodeHandle,
+ // newLiaisonHandle) instead of going through pkg/test/setup, so it has no
+ // awaitPreloadedSchemasApplied-style barrier: PreloadSchemaViaProperty runs after
+ // the liaison is already serving, and even once the schema lands server-side the
+ // liaison's own write cache can still lag for a moment. A lagging liaison refuses
+ // the single row with a status (e.g. STATUS_NOT_FOUND) rather than a stream error,
+ // so a prior Recv()-only check that ignored the status would silently accept a
+ // refused write. A refused row stores nothing, so retrying the whole
+ // open-stream-send-close-check sequence is safe; baseTime is fixed across
+ // attempts since it is also the timestamp the caller later queries by, and it
+ // stays well within the timestamp validator's wide allowed range for the
+ // duration of a single EventuallyTimeout window.
+ Eventually(func() (string, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ stream, err := client.Write(ctx)
+ if err != nil {
+ return "", err
+ }
+ if sendErr := stream.Send(&tracev1.WriteRequest{
+ Metadata: &commonv1.Metadata{Name: "sw", Group: "test-trace-group"},
+ Tags: tags,
+ Span: []byte("span-" + traceID),
+ Version: 1,
+ }); sendErr != nil {
+ return "", sendErr
+ }
+ if closeErr := stream.CloseSend(); closeErr != nil {
+ return "", closeErr
+ }
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ return "", recvErr
+ }
+ return resp.GetStatus(), nil
+ }, flags.EventuallyTimeout).Should(Equal(modelv1.Status_STATUS_SUCCEED.String()))
return baseTime
}
diff --git a/test/integration/standalone/backup/common.go b/test/integration/standalone/backup/common.go
index a19984efe..1b99d532e 100644
--- a/test/integration/standalone/backup/common.go
+++ b/test/integration/standalone/backup/common.go
@@ -146,16 +146,25 @@ func initializeTestData(conn *grpc.ClientConn) error {
Name: "p",
Group: "g",
}
- resp, err := client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
- Metadata: md,
- Id: "1",
- Tags: []*modelv1.Tag{
- {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
- },
- }})
- if err != nil {
- return err
+ // A just-created group and property reach the liaison's in-memory repositories
+ // asynchronously via the schema watch, so the first Apply right after creating them
+ // can transiently fail. EventuallyErr rides out that propagation window while still
+ // returning a plain error, matching this function's error-chaining contract.
+ var resp *propertyv1.ApplyResponse
+ applyErr := test.EventuallyErr(func(g gomega.Gomega) {
+ var err error
+ resp, err = client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
+ Metadata: md,
+ Id: "1",
+ Tags: []*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
+ },
+ }})
+ g.Expect(err).NotTo(gomega.HaveOccurred())
+ })
+ if applyErr != nil {
+ return applyErr
}
gomega.Expect(resp.Created).To(gomega.BeTrue())
diff --git a/test/integration/standalone/deletion/common.go b/test/integration/standalone/deletion/common.go
index f39fd57c5..9906dd930 100644
--- a/test/integration/standalone/deletion/common.go
+++ b/test/integration/standalone/deletion/common.go
@@ -34,12 +34,14 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ schemav1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/schema/v1"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -91,6 +93,7 @@ var _ = g.Describe("GroupDeletion", func() {
var groupClient databasev1.GroupRegistryServiceClient
var measureRegClient databasev1.MeasureRegistryServiceClient
var measureWriteClient measurev1.MeasureServiceClient
+ var measureBarrierClient schemav1.SchemaBarrierServiceClient
g.BeforeEach(func() {
goods = gleak.Goroutines()
@@ -102,6 +105,7 @@ var _ = g.Describe("GroupDeletion", func() {
groupClient = databasev1.NewGroupRegistryServiceClient(conn)
measureRegClient = databasev1.NewMeasureRegistryServiceClient(conn)
measureWriteClient = measurev1.NewMeasureServiceClient(conn)
+ measureBarrierClient = schemav1.NewSchemaBarrierServiceClient(conn)
})
g.AfterEach(func() {
@@ -200,7 +204,7 @@ var _ = g.Describe("GroupDeletion", func() {
},
})
gm.Expect(err).ShouldNot(gm.HaveOccurred())
- createMeasureInGroup(measureRegClient, groupName, measureName)
+ createMeasureInGroup(measureRegClient, measureBarrierClient, groupName, measureName)
writeMeasureDataToGroup(measureWriteClient, groupName, measureName, 5)
g.By("Verifying group data directory exists on disk")
@@ -267,7 +271,7 @@ var _ = g.Describe("GroupDeletion", func() {
},
})
gm.Expect(err).ShouldNot(gm.HaveOccurred())
- createMeasureInGroup(measureRegClient, groupName, measureName)
+ createMeasureInGroup(measureRegClient, measureBarrierClient, groupName, measureName)
writeMeasureDataToGroup(measureWriteClient, groupName, measureName, 5)
g.By("Deleting the group with force=true")
@@ -309,7 +313,7 @@ var _ = g.Describe("GroupDeletion", func() {
})
return createErr
}, flags.EventuallyTimeout).Should(gm.Succeed())
- createMeasureInGroup(measureRegClient, groupName, measureName)
+ createMeasureInGroup(measureRegClient, measureBarrierClient, groupName, measureName)
writeMeasureDataToGroup(measureWriteClient, groupName, measureName, 5)
g.By("Verifying the recreated group data directory exists on disk")
@@ -345,8 +349,20 @@ var _ = g.Describe("GroupDeletion", func() {
})
})
-func createMeasureInGroup(regClient databasev1.MeasureRegistryServiceClient, group, measure string) {
- _, createErr := regClient.Create(context.TODO(), &databasev1.MeasureRegistryServiceCreateRequest{
+// createMeasureInGroup creates measure in group and waits for the cluster's schema
+// watermark to reach the Create response's mod_revision before returning.
+//
+// AwaitRevisionApplied is a hint under eventual consistency: it confirms the cluster
+// watermark, not that this liaison's own entity cache has applied the schema yet.
+// writeMeasureDataToGroup retries a refused write to tolerate that residual per-liaison
+// lag, so the barrier only needs to replace the blind sleep this used to rely on for the
+// common case.
+func createMeasureInGroup(
+ regClient databasev1.MeasureRegistryServiceClient,
+ barrierClient schemav1.SchemaBarrierServiceClient,
+ group, measure string,
+) {
+ createResp, createErr := regClient.Create(context.TODO(), &databasev1.MeasureRegistryServiceCreateRequest{
Measure: &databasev1.Measure{
Metadata: &commonv1.Metadata{Name: measure, Group: group},
Entity: &databasev1.Entity{TagNames: []string{"id"}},
@@ -363,16 +379,25 @@ func createMeasureInGroup(regClient databasev1.MeasureRegistryServiceClient, gro
},
})
gm.Expect(createErr).ShouldNot(gm.HaveOccurred())
- time.Sleep(2 * time.Second)
+ barrierResp, barrierErr := barrierClient.AwaitRevisionApplied(context.TODO(), &schemav1.AwaitRevisionAppliedRequest{
+ MinRevision: createResp.GetModRevision(),
+ Timeout: durationpb.New(flags.EventuallyTimeout),
+ })
+ gm.Expect(barrierErr).ShouldNot(gm.HaveOccurred())
+ gm.Expect(barrierResp.GetApplied()).Should(gm.BeTrue(), "timeout waiting for schema revision %d", createResp.GetModRevision())
}
-func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) {
+// sendMeasureBatchToGroup writes count distinct measure rows on one write stream and returns
+// the status string of every response, in send order.
+func sendMeasureBatchToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) ([]string, error) {
wc, wcErr := writeClient.Write(context.TODO())
- gm.Expect(wcErr).ShouldNot(gm.HaveOccurred())
+ if wcErr != nil {
+ return nil, wcErr
+ }
metadata := &commonv1.Metadata{Name: measure, Group: group}
baseTime := time.Now().Truncate(time.Millisecond)
for idx := 0; idx < count; idx++ {
- sendErr := wc.Send(&measurev1.WriteRequest{
+ if sendErr := wc.Send(&measurev1.WriteRequest{
Metadata: metadata,
DataPoint: &measurev1.DataPointValue{
Timestamp: timestamppb.New(baseTime.Add(time.Duration(idx) * time.Second)),
@@ -386,16 +411,44 @@ func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group,
}},
},
MessageId: uint64(time.Now().UnixNano() + int64(idx)),
- })
- gm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ }); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeSendErr := wc.CloseSend(); closeSendErr != nil {
+ return nil, closeSendErr
}
- closeSendErr := wc.CloseSend()
- gm.Expect(closeSendErr).ShouldNot(gm.HaveOccurred())
+ var statuses []string
for {
- _, recvErr := wc.Recv()
+ resp, recvErr := wc.Recv()
if errors.Is(recvErr, io.EOF) {
break
}
- gm.Expect(recvErr).ShouldNot(gm.HaveOccurred())
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
}
+ return statuses, nil
+}
+
+// writeMeasureDataToGroup retries sendMeasureBatchToGroup until every row is accepted. Rows can be
+// refused by status until the liaison's schema cache catches up. A measure row is keyed by
+// series and timestamp with the latest version winning, so resending the batch with the same
+// values is idempotent: any refusal, including a partly accepted batch, is retried within
+// flags.EventuallyTimeout.
+func writeMeasureDataToGroup(writeClient measurev1.MeasureServiceClient, group, measure string, count int) {
+ gm.Eventually(func(innerGm gm.Gomega) {
+ statuses, sendErr := sendMeasureBatchToGroup(writeClient, group, measure, count)
+ innerGm.Expect(sendErr).ShouldNot(gm.HaveOccurred())
+ innerGm.Expect(statuses).Should(gm.HaveLen(count), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ innerGm.Expect(succeeded).Should(gm.Equal(count),
+ "measure write to %s/%s rejected; statuses=%v", group, measure, statuses)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
diff --git a/test/integration/standalone/inspection/common.go b/test/integration/standalone/inspection/common.go
index fade0b697..48cc103c7 100644
--- a/test/integration/standalone/inspection/common.go
+++ b/test/integration/standalone/inspection/common.go
@@ -20,6 +20,7 @@ package inspection
import (
"context"
+ "errors"
"fmt"
"io"
"strconv"
@@ -109,10 +110,13 @@ var _ = ginkgo.ReportAfterSuite("Standalone Inspect Suite", func(report ginkgo.R
}
})
-func writeMeasureData(ctx context.Context, groupName, measureName string, dataCount int) {
+// sendMeasureDataBatch writes dataCount distinct measure rows on one write stream and
+// returns the status string of every response, in send order.
+func sendMeasureDataBatch(ctx context.Context, groupName, measureName string, dataCount int) ([]string, error) {
writeClient, writeErr := measureWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
-
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: measureName,
Group: groupName,
@@ -134,20 +138,55 @@ func writeMeasureData(ctx context.Context, groupName, measureName string, dataCo
},
MessageId: uint64(time.Now().UnixNano()),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
+ }
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
+ }
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ return statuses, nil
}
-func writeStreamData(ctx context.Context, groupName, streamName string, dataCount int) {
- writeClient, writeErr := streamWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
+// writeMeasureData retries sendMeasureDataBatch until every row is accepted. Rows can be
+// refused by status until the liaison's schema cache catches up. A measure row is keyed by
+// series and timestamp with the latest version winning, so resending the batch with the same
+// values is idempotent: any refusal, including a partly accepted batch, is retried within
+// flags.EventuallyTimeout.
+func writeMeasureData(ctx context.Context, groupName, measureName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendMeasureDataBatch(ctx, groupName, measureName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "measure write to %s/%s rejected; statuses=%v", groupName, measureName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+}
+// sendStreamDataBatch writes dataCount distinct stream rows on one write stream and returns
+// the status string of every response, in send order.
+func sendStreamDataBatch(ctx context.Context, groupName, streamName string, dataCount int) ([]string, error) {
+ writeClient, writeErr := streamWriteClient.Write(ctx)
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: streamName,
Group: groupName,
@@ -167,20 +206,58 @@ func writeStreamData(ctx context.Context, groupName, streamName string, dataCoun
},
MessageId: uint64(time.Now().UnixNano()),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
+ }
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
+ }
+ return statuses, nil
}
-func writeTraceData(ctx context.Context, groupName, traceName string, dataCount int) {
- writeClient, writeErr := traceWriteClient.Write(ctx)
- gomega.Expect(writeErr).ShouldNot(gomega.HaveOccurred())
+// writeStreamData retries sendStreamDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag behind the group/schema it was just given, and a
+// refused row stores nothing, so repeating the whole batch is safe. If some rows succeed and
+// some are refused, the batch cannot be safely repeated without duplicating the accepted
+// rows, so it stops retrying at once.
+func writeStreamData(ctx context.Context, groupName, streamName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendStreamDataBatch(ctx, groupName, streamName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < dataCount {
+ gomega.StopTrying(fmt.Sprintf("stream write to %s/%s partially rejected; statuses=%v", groupName, streamName, statuses)).Now()
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "stream write to %s/%s rejected; statuses=%v", groupName, streamName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
+}
+// sendTraceDataBatch writes dataCount distinct trace rows on one write stream and returns the
+// status string of every response, in send order.
+func sendTraceDataBatch(ctx context.Context, groupName, traceName string, dataCount int) ([]string, error) {
+ writeClient, writeErr := traceWriteClient.Write(ctx)
+ if writeErr != nil {
+ return nil, writeErr
+ }
metadata := &commonv1.Metadata{
Name: traceName,
Group: groupName,
@@ -201,14 +278,49 @@ func writeTraceData(ctx context.Context, groupName, traceName string, dataCount
Span: []byte(fmt.Sprintf("span_data_%d", idx)),
Version: uint64(idx + 1),
}
- sendErr := writeClient.Send(req)
- gomega.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ if sendErr := writeClient.Send(req); sendErr != nil {
+ return nil, sendErr
+ }
}
- gomega.Expect(writeClient.CloseSend()).To(gomega.Succeed())
- gomega.Eventually(func() error {
- _, recvErr := writeClient.Recv()
- return recvErr
- }, flags.EventuallyTimeout).Should(gomega.Equal(io.EOF))
+ if closeErr := writeClient.CloseSend(); closeErr != nil {
+ return nil, closeErr
+ }
+ var statuses []string
+ for {
+ resp, recvErr := writeClient.Recv()
+ if errors.Is(recvErr, io.EOF) {
+ break
+ }
+ if recvErr != nil {
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
+ }
+ return statuses, nil
+}
+
+// writeTraceData retries sendTraceDataBatch while every row in the batch is refused — the
+// liaison's write cache can still lag behind the group/schema it was just given, and a
+// refused row stores nothing, so repeating the whole batch is safe. If some rows succeed and
+// some are refused, the batch cannot be safely repeated without duplicating the accepted
+// rows, so it stops retrying at once.
+func writeTraceData(ctx context.Context, groupName, traceName string, dataCount int) {
+ gomega.Eventually(func(g gomega.Gomega) {
+ statuses, sendErr := sendTraceDataBatch(ctx, groupName, traceName, dataCount)
+ g.Expect(sendErr).ShouldNot(gomega.HaveOccurred())
+ g.Expect(statuses).Should(gomega.HaveLen(dataCount), "one write response per row")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ if succeeded > 0 && succeeded < dataCount {
+ gomega.StopTrying(fmt.Sprintf("trace write to %s/%s partially rejected; statuses=%v", groupName, traceName, statuses)).Now()
+ }
+ g.Expect(succeeded).Should(gomega.Equal(dataCount),
+ "trace write to %s/%s rejected; statuses=%v", groupName, traceName, statuses)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
}
var _ = ginkgo.Describe("Inspect measure in standalone mode", ginkgo.Ordered, func() {
@@ -309,22 +421,26 @@ var _ = ginkgo.Describe("Inspect measure in standalone mode", ginkgo.Ordered, fu
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
- ginkgo.By("Verifying data info collected")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "data node %d should have data size > 0", idx)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total data size: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // measure may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+ ginkgo.By("Verifying data info collected")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "data node %d should have data size > 0", idx)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total data size: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return liaison info", func() {
@@ -416,22 +532,26 @@ var _ = ginkgo.Describe("Inspect stream in standalone mode", ginkgo.Ordered, fun
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting stream group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
- ginkgo.By("Verifying stream data info collected")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "stream data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "stream data node %d should have data size > 0", idx)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Stream data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total stream data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total stream data size: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // stream may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting stream group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+ ginkgo.By("Verifying stream data info collected")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "stream data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "stream data node %d should have data size > 0", idx)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Stream data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total stream data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total stream data size: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return schema info", func() {
@@ -525,22 +645,26 @@ var _ = ginkgo.Describe("Inspect trace in standalone mode", ginkgo.Ordered, func
})
ginkgo.It("should return data info", func() {
- ginkgo.By("Inspecting trace group")
- resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
- gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
- gomega.Expect(resp).NotTo(gomega.BeNil())
- ginkgo.By("Verifying trace data info collected")
- gomega.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
- var totalDataSize int64
- for idx, dataInfo := range resp.DataInfo {
- gomega.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "trace data node %d should have node info", idx)
- gomega.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "trace data node %d should have data size > 0", idx)
- totalDataSize += dataInfo.DataSizeBytes
- ginkgo.GinkgoWriter.Printf("Trace data node %d: %s, segments: %d, size: %d bytes\n",
- idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
- }
- gomega.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total trace data size should be > 0")
- ginkgo.GinkgoWriter.Printf("Total trace data size: %d bytes\n", totalDataSize)
+ // Inspect can race the write path's flush/sync to disk, so a freshly written
+ // trace may transiently report a zero-size node; retry until it settles.
+ gomega.Eventually(func(g gomega.Gomega) {
+ ginkgo.By("Inspecting trace group")
+ resp, err := groupClient.Inspect(ctx, &databasev1.GroupRegistryServiceInspectRequest{Group: groupName})
+ g.Expect(err).ShouldNot(gomega.HaveOccurred())
+ g.Expect(resp).NotTo(gomega.BeNil())
+ ginkgo.By("Verifying trace data info collected")
+ g.Expect(len(resp.DataInfo)).Should(gomega.BeNumerically(">=", 1), "should collect from standalone node")
+ var totalDataSize int64
+ for idx, dataInfo := range resp.DataInfo {
+ g.Expect(dataInfo.Node).NotTo(gomega.BeNil(), "trace data node %d should have node info", idx)
+ g.Expect(dataInfo.DataSizeBytes).Should(gomega.BeNumerically(">", 0), "trace data node %d should have data size > 0", idx)
+ totalDataSize += dataInfo.DataSizeBytes
+ ginkgo.GinkgoWriter.Printf("Trace data node %d: %s, segments: %d, size: %d bytes\n",
+ idx, dataInfo.Node.Metadata.Name, len(dataInfo.SegmentInfo), dataInfo.DataSizeBytes)
+ }
+ g.Expect(totalDataSize).Should(gomega.BeNumerically(">", 0), "total trace data size should be > 0")
+ ginkgo.GinkgoWriter.Printf("Total trace data size: %d bytes\n", totalDataSize)
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
})
ginkgo.It("should return schema info", func() {
diff --git a/test/integration/standalone/other/rbac_data/rbac_data_test.go b/test/integration/standalone/other/rbac_data/rbac_data_test.go
index 67ec6c592..f0436cda8 100644
--- a/test/integration/standalone/other/rbac_data/rbac_data_test.go
+++ b/test/integration/standalone/other/rbac_data/rbac_data_test.go
@@ -283,34 +283,75 @@ func metricValue(addr, name string, labels ...string) (float64, error) {
return 0, nil
}
-// sendMeasureFrames opens a real bidirectional write, sends every frame, closes the send side
-// and returns the terminal status of the stream. A per-frame denial surfaces on Recv rather
-// than on Send, so the send side has to be closed before the status can be read.
-func sendMeasureFrames(a actor, client measurev1.MeasureServiceClient, frames ...*measurev1.WriteRequest) error {
+// sendMeasureFrameStatuses opens a real bidirectional write, sends every frame, closes the
+// send side, and returns the status string of each response in send order. A transport-level
+// error (e.g. codes.PermissionDenied for a refused stream) ends the stream before any status
+// is returned.
+func sendMeasureFrameStatuses(a actor, client measurev1.MeasureServiceClient, frames ...*measurev1.WriteRequest) ([]string, error) {
stream, openErr := client.Write(a.ctx())
if openErr != nil {
- return openErr
+ return nil, openErr
}
for _, frame := range frames {
if sendErr := stream.Send(frame); sendErr != nil {
if errors.Is(sendErr, io.EOF) {
break
}
- return sendErr
+ return nil, sendErr
}
}
if closeErr := stream.CloseSend(); closeErr != nil {
- return closeErr
+ return nil, closeErr
}
+ var statuses []string
for {
- _, recvErr := stream.Recv()
+ resp, recvErr := stream.Recv()
if errors.Is(recvErr, io.EOF) {
- return nil
+ return statuses, nil
}
if recvErr != nil {
- return recvErr
+ return statuses, recvErr
+ }
+ statuses = append(statuses, resp.GetStatus())
+ }
+}
+
+// sendMeasureFrames collapses sendMeasureFrameStatuses to a single error: nil only when every
+// frame returned STATUS_SUCCEED. It is a single attempt with no retry, for call sites whose
+// point is the terminal condition itself (e.g. an RBAC refusal, which already ends the stream
+// with a transport-level error rather than a per-frame status).
+func sendMeasureFrames(a actor, client measurev1.MeasureServiceClient, frames ...*measurev1.WriteRequest) error {
+ statuses, err := sendMeasureFrameStatuses(a, client, frames...)
+ if err != nil {
+ return err
+ }
+ for _, writeStatus := range statuses {
+ if writeStatus != modelv1.Status_STATUS_SUCCEED.String() {
+ return fmt.Errorf("write frame rejected with status %s", writeStatus)
}
}
+ return nil
+}
+
+// seedMeasureFrames sends frames and retries until every frame is accepted.
+// AwaitSchemaApplied in the suite's BeforeEach only confirms the schema registry; this
+// liaison's own entity cache can still lag it for a moment, refusing a frame by status (not by
+// a stream error) until it catches up. A refused frame stores nothing, and a measure row is keyed
+// by series and timestamp with the latest version winning, so resending the whole batch, even a
+// partly accepted one, is idempotent.
+func seedMeasureFrames(a actor, client measurev1.MeasureServiceClient, frames ...*measurev1.WriteRequest) {
+ gm.EventuallyWithOffset(1, func(innerGm gm.Gomega) {
+ statuses, sendErr := sendMeasureFrameStatuses(a, client, frames...)
+ innerGm.Expect(sendErr).NotTo(gm.HaveOccurred())
+ innerGm.Expect(statuses).To(gm.HaveLen(len(frames)), "one write response per frame")
+ succeeded := 0
+ for _, writeStatus := range statuses {
+ if writeStatus == modelv1.Status_STATUS_SUCCEED.String() {
+ succeeded++
+ }
+ }
+ innerGm.Expect(succeeded).To(gm.Equal(len(frames)), "seed write by %s rejected; statuses=%v", a.name, statuses)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
}
// httpCall issues one grpc-gateway request with Basic credentials and returns the status code
@@ -437,10 +478,10 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
// group, is refused another, and is refused a request that mixes the two rather than being
// served the part of it it is allowed. The expected codes are read off issue #14016's R1.
g.It("serves a group-scoped reader its own markers and refuses every mixed request", func() {
- gm.Expect(sendMeasureFrames(adminActor, measures,
+ seedMeasureFrames(adminActor, measures,
markerFrame(groupAlpha, markerAlpha, baseTime, 1),
markerFrame(groupBeta, markerBeta, baseTime, 2),
- )).To(gm.Succeed(), "the administrator must be allowed to seed both groups' markers")
+ )
gm.Eventually(func() []string {
resp, queryErr := measures.Query(adminActor.ctx(), markerQuery(baseTime, groupAlpha, groupBeta))
@@ -500,9 +541,7 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
// resource-bearing frame is decided against the group it resolves to, and a frame that is
// refused never reaches storage — which an administrator's query afterwards is what proves.
g.It("refuses a forbidden write frame mid-stream and stores nothing from it", func() {
- gm.Expect(sendMeasureFrames(writerAlphaActor, measures,
- markerFrame(groupAlpha, markerAlpha, baseTime, 1),
- )).To(gm.Succeed(), "the alpha writer must be allowed to write its own group")
+ seedMeasureFrames(writerAlphaActor, measures, markerFrame(groupAlpha, markerAlpha, baseTime, 1))
gm.Eventually(func() []string {
resp, queryErr := measures.Query(adminActor.ctx(), markerQuery(baseTime, groupAlpha))
@@ -519,8 +558,16 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
gm.Expect(status.Code(forbiddenErr)).To(gm.Equal(codes.PermissionDenied),
"a frame addressing a forbidden group must end the stream with PermissionDenied")
- betaResp, betaErr := measures.Query(adminActor.ctx(), markerQuery(baseTime.Add(time.Second), groupBeta))
- gm.Expect(betaErr).NotTo(gm.HaveOccurred(), "the administrator must be able to look for the refused frame")
+ // A just-created group's shard can still be opening even after the BeforeEach schema
+ // barrier returns, in which case the query plan transiently fails with "group not
+ // found" rather than answering empty. Retry on error only; a genuine leak still
+ // surfaces as a non-empty successful response, which the assertion below still catches.
+ var betaResp *measurev1.QueryResponse
+ gm.Eventually(func() error {
+ var queryErr error
+ betaResp, queryErr = measures.Query(adminActor.ctx(), markerQuery(baseTime.Add(time.Second), groupBeta))
+ return queryErr
+ }, flags.EventuallyTimeout).ShouldNot(gm.HaveOccurred(), "the administrator must be able to look for the refused frame")
gm.Expect(markersIn(betaResp)).To(gm.BeEmpty(), "a refused frame must leave nothing in storage")
readerErr := sendMeasureFrames(readerAlphaActor, measures, markerFrame(groupAlpha, markerAlpha, baseTime, 4))
@@ -532,10 +579,22 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
// connection observes a live revocation, and the next frame on a write stream that was
// admitted under the previous revision is denied before it can reach storage.
g.It("applies live revocation to unary data calls and an already-open write stream", func() {
+ allowedFrame := markerFrame(groupAlpha, markerAlpha, baseTime, 1)
+ // AwaitSchemaApplied in the BeforeEach only confirms the schema registry; this liaison's
+ // own entity cache can still lag it for a moment, refusing the frame by status. Confirm
+ // it lands using the same all-refused-retry helper as the other specs' seed writes, on
+ // a throwaway stream that is opened and closed as part of that helper.
+ seedMeasureFrames(writerAlphaActor, measures, allowedFrame)
+
+ // The persistent stream used for the rest of this spec (reused below to send a second
+ // frame after revocation) is opened separately: a per-frame WriteResponse on THIS stream
+ // is deferred until it closes, which this spec never does before reusing it, so there is
+ // nothing to retry here. Resending the identical frame is safe — a measure row is keyed
+ // by (entity, timestamp) with last-version-wins dedup enforced even at query time against
+ // still-unmerged parts, not just at compaction — and seedMeasureFrames above already
+ // guarantees the row is stored regardless of how this particular send resolves.
writeStream, openErr := measures.Write(writerAlphaActor.ctx())
gm.Expect(openErr).NotTo(gm.HaveOccurred())
-
- allowedFrame := markerFrame(groupAlpha, markerAlpha, baseTime, 1)
gm.Expect(writeStream.Send(allowedFrame)).To(gm.Succeed(),
"the writer must send an allowed frame before its grant is revoked")
gm.Eventually(func() (float64, error) {
@@ -545,8 +604,11 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
}, flags.EventuallyTimeout).Should(gm.BeNumerically(">=", 1),
"the open stream's first frame must be allowed under the original revision")
- _, allowedQueryErr := measures.Query(readerAlphaActor.ctx(), markerQuery(baseTime, groupAlpha))
- gm.Expect(allowedQueryErr).NotTo(gm.HaveOccurred(),
+ // Same just-opened-shard race as above: retry on error only.
+ gm.Eventually(func() error {
+ _, queryErr := measures.Query(readerAlphaActor.ctx(), markerQuery(baseTime, groupAlpha))
+ return queryErr
+ }, flags.EventuallyTimeout).ShouldNot(gm.HaveOccurred(),
"the reader must be allowed on the existing connection before its grant is revoked")
replacePolicy(policyFile, revokedDataPolicy)
@@ -608,10 +670,17 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
gm.Expect(status.Code(betaApplyErr)).To(gm.Equal(codes.PermissionDenied),
"the alpha writer must not apply a property in beta")
- absent, absentErr := properties.Query(adminActor.ctx(), &propertyv1.QueryRequest{
- Groups: []string{groupAlphaProp, groupBetaProp}, Name: fixtureProp, Ids: []string{"1"},
- })
- gm.Expect(absentErr).NotTo(gm.HaveOccurred())
+ // The BeforeEach barrier only awaits the group and measure schema keys, not the two
+ // property schemas it also creates; this query can transiently fail until this
+ // liaison's cache has them. Retry on error only.
+ var absent *propertyv1.QueryResponse
+ gm.Eventually(func() error {
+ var queryErr error
+ absent, queryErr = properties.Query(adminActor.ctx(), &propertyv1.QueryRequest{
+ Groups: []string{groupAlphaProp, groupBetaProp}, Name: fixtureProp, Ids: []string{"1"},
+ })
+ return queryErr
+ }, flags.EventuallyTimeout).ShouldNot(gm.HaveOccurred())
gm.Expect(absent.GetProperties()).To(gm.BeEmpty(), "a refused apply must leave the property absent")
gm.Eventually(func() error {
@@ -646,10 +715,10 @@ var _ = g.Describe("rbac-data group-scoped data authorization through the real l
// casing, its comments, its parameters and the route that carried it are not resources, so
// none of them can address a group the decision does not see.
g.It("decides a ByDBQL query by the groups of the request it transformed into", func() {
- gm.Expect(sendMeasureFrames(adminActor, measures,
+ seedMeasureFrames(adminActor, measures,
markerFrame(groupAlpha, markerAlpha, baseTime, 1),
markerFrame(groupBeta, markerBeta, baseTime, 2),
- )).To(gm.Succeed(), "the administrator must be allowed to seed both groups' markers")
+ )
allowed := fmt.Sprintf("SELECT * FROM MEASURE %s IN %s", fixtureMeasure, groupAlpha)
_, allowedErr := bydbql.Query(readerAlphaActor.ctx(), &bydbqlv1.QueryRequest{Query: allowed})
diff --git a/test/property_repair/shared_utils.go b/test/property_repair/shared_utils.go
index 2d3fb3591..7dd296ecc 100644
--- a/test/property_repair/shared_utils.go
+++ b/test/property_repair/shared_utils.go
@@ -42,6 +42,7 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
)
// Constants for property repair performance testing.
@@ -228,20 +229,28 @@ func WriteProperties(ctx context.Context, propertyServiceClient propertyv1.Prope
largeData := GenerateLargeData(DataSize)
timestamp := time.Now().Format(time.RFC3339Nano)
- _, writeErr := propertyServiceClient.Apply(ctx, &propertyv1.ApplyRequest{
- Property: &propertyv1.Property{
- Metadata: &commonv1.Metadata{
- Name: PropertyName,
- Group: GroupName,
+ // A just-created group and property schema reach the liaison's in-memory
+ // repositories asynchronously via the schema watch, so an Apply issued soon
+ // after CreateGroup/CreatePropertySchema can transiently refuse the write.
+ // Eventually rides out that propagation window; GinkgoRecover (deferred above)
+ // catches the failure if it never converges.
+ gomega.Eventually(func() error {
+ var writeErr error
+ _, writeErr = propertyServiceClient.Apply(ctx, &propertyv1.ApplyRequest{
+ Property: &propertyv1.Property{
+ Metadata: &commonv1.Metadata{
+ Name: PropertyName,
+ Group: GroupName,
+ },
+ Id: propertyID,
+ Tags: []*modelv1.Tag{
+ {Key: "data", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: largeData}}}},
+ {Key: "timestamp", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: timestamp}}}},
+ },
},
- Id: propertyID,
- Tags: []*modelv1.Tag{
- {Key: "data", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: largeData}}}},
- {Key: "timestamp", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: timestamp}}}},
- },
- },
- })
- gomega.Expect(writeErr).NotTo(gomega.HaveOccurred())
+ })
+ return writeErr
+ }, flags.EventuallyTimeout).Should(gomega.Succeed())
count++
atomic.AddInt64(&totalProcessed, 1)