You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/10/07 10:05:20 UTC
(skywalking-banyandb) branch main updated: feat(index): cut the series index over to the native index and remove the legacy engine (NIDX-03) (#1397)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 8c1723643 feat(index): cut the series index over to the native index and remove the legacy engine (NIDX-03) (#1397)
8c1723643 is described below
commit 8c1723643fd39cf930ec4653797d4ac79ca4af63
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Oct 7 18:05:14 2026 +0800
feat(index): cut the series index over to the native index and remove the legacy engine (NIDX-03) (#1397)
---
CHANGES.md | 1 +
banyand/backup/lifecycle/roundtrip_test.go | 4 +-
banyand/cmd/dump/property.go | 313 ++---
banyand/cmd/dump/property_test.go | 226 ++-
banyand/cmd/dump/sidx.go | 92 +-
banyand/cmd/dump/sidx_test.go | 114 ++
banyand/internal/dump/helpers.go | 40 +-
banyand/internal/dump/measure/suite_test.go | 4 +-
banyand/internal/migration/fsutil.go | 9 +-
banyand/internal/migration/unionsidx.go | 150 +-
.../migration/unionsidx_native_source_test.go | 49 +-
banyand/internal/migration/unionsidx_test.go | 66 +
banyand/internal/storage/index.go | 687 +++++++---
.../storage/index_async_persist_failure_test.go | 113 ++
banyand/internal/storage/index_crash_cut_test.go | 150 ++
.../storage/index_external_segments_test.go | 239 ++++
.../storage/index_fixture_generate_test.go | 206 +++
.../internal/storage/index_fixture_verify_test.go | 310 +++++
.../internal/storage/index_reference_model_test.go | 526 +++++++
banyand/internal/storage/index_semantics_test.go | 242 ++++
banyand/internal/storage/index_test.go | 10 +-
banyand/internal/storage/nidx03_benchmark_test.go | 541 ++++++++
.../storage/nidx03_dependency_guard_test.go | 334 +++++
.../internal/storage/nidx03_rollback_write_test.go | 514 +++++++
banyand/internal/storage/rotation_test.go | 3 +
banyand/internal/storage/segment.go | 174 ++-
.../storage/segment_close_resources_test.go | 105 ++
.../internal/storage/segment_idle_reclaim_test.go | 1 +
.../storage/segment_snapshot_closed_native_test.go | 237 ++++
banyand/internal/storage/segment_test.go | 12 +
.../storage/snapshot_closed_segment_test.go | 4 +-
banyand/internal/storage/storage.go | 47 +-
.../nidx03_fixture/indexmode/sidx/000000000001.snp | Bin 0 -> 48 bytes
.../nidx03_fixture/indexmode/sidx/000000000002.seg | Bin 0 -> 2232 bytes
.../nidx03_fixture/indexmode/sidx/000000000002.snp | Bin 0 -> 48 bytes
.../nidx03_fixture/normal/sidx/000000000001.snp | Bin 0 -> 48 bytes
.../nidx03_fixture/normal/sidx/000000000002.seg | Bin 0 -> 3049 bytes
.../nidx03_fixture/normal/sidx/000000000002.snp | Bin 0 -> 48 bytes
banyand/internal/storage/tsdb.go | 6 +-
banyand/internal/storage/tsdb_test.go | 28 +
banyand/internal/storage/visitor_test.go | 7 +
banyand/measure/cache_benchmark_test.go | 5 +
banyand/measure/metadata.go | 11 +-
banyand/measure/migration_indexmode_copy.go | 141 +-
banyand/measure/migration_indexmode_copy_test.go | 176 ++-
.../migration_indexmode_native_source_test.go | 107 +-
banyand/measure/migration_verify.go | 100 +-
banyand/measure/query.go | 21 +-
banyand/measure/write_data_segmentref_test.go | 11 +-
.../schema/reader/e2e_native_schema_walk_test.go | 49 +-
banyand/metadata/schema/reader/reader.go | 12 +-
banyand/metadata/schema/schemaserver/service.go | 1 -
banyand/property/db/db.go | 223 +--
banyand/property/db/db_lifetime_test.go | 238 ++--
.../db/e2e_native_legacy_acceptance_test.go | 61 +-
.../property/db/e2e_native_writer_cutover_test.go | 105 +-
banyand/property/db/native_expiry_test.go | 2 +-
banyand/property/db/native_property.go | 184 +--
banyand/property/db/native_property_match_test.go | 129 +-
.../db/nidx03_property_rollback_write_test.go | 129 ++
.../property/db/nidx03_property_upgrade_test.go | 191 +++
banyand/property/db/ownership_lifecycle_test.go | 163 ---
banyand/property/db/query_capacity.go | 7 -
banyand/property/db/query_capacity_test.go | 29 +-
banyand/property/db/repair.go | 8 +-
banyand/property/db/repair_gossip.go | 23 +-
banyand/property/db/repair_gossip_test.go | 68 +-
banyand/property/db/repair_test.go | 4 +-
banyand/property/db/schema_workload_bench_test.go | 35 +-
banyand/property/db/shard.go | 354 +----
banyand/property/db/shard_test.go | 33 +-
banyand/property/service.go | 1 -
banyand/stream/index.go | 8 +-
banyand/stream/metrics.go | 8 +-
banyand/stream/migration_verify.go | 34 +-
.../stream/native_element_index_rollback_test.go | 140 ++
banyand/stream/tstable.go | 13 +-
banyand/trace/finalize_scanner_test.go | 4 +
banyand/trace/metadata.go | 10 +-
banyand/trace/metrics.go | 8 +-
banyand/trace/write_data_segmentref_test.go | 11 +-
bydbctl/internal/cmd/property_test.go | 100 +-
docs/concept/storage-and-format.md | 44 +-
docs/design/0.12.0/data-export-import/README.md | 20 +
.../native-inverted-index/nidx-03-series-index.md | 469 +++++++
.../verification/nidx-03-series-cutover/README.md | 416 ++++++
.../nativeice/dictionary_regression_test.go | 40 +
pkg/index/internal/nativeice/encode.go | 7 +
.../nativeice/identifier_docvalues_test.go | 173 +++
pkg/index/internal/nativeice/merge.go | 84 +-
.../internal/nativeice/prepare_term_filter_test.go | 84 ++
pkg/index/internal/nativeice/selection.go | 19 +
pkg/index/inverted/inverted.go | 331 +----
pkg/index/inverted/inverted_series.go | 312 +----
pkg/index/inverted/inverted_series_test.go | 915 -------------
pkg/index/inverted/inverted_test.go | 15 -
.../inverted/legacy_merge_field_order_test.go | 9 +-
.../inverted/native_adapter_legacy_compat_test.go | 3 +-
pkg/index/inverted/native_childproc_test.go | 8 +-
pkg/index/inverted/native_dependency_test.go | 188 ---
pkg/index/inverted/native_fixture_nidx01e_test.go | 3 +-
pkg/index/inverted/native_fixture_nidx02a_test.go | 3 +-
pkg/index/inverted/native_fixture_test.go | 2 +-
.../native_frequency_legacy_compat_test.go | 5 +-
pkg/index/inverted/native_multichunk_walk_test.go | 3 +-
pkg/index/inverted/native_nidx01b_boundary_test.go | 265 ----
pkg/index/inverted/native_nidx01c_walk_test.go | 58 +-
.../inverted/native_nidx01d_selection_test.go | 96 +-
.../inverted/native_nidx01e_tuple_page_test.go | 251 +---
pkg/index/inverted/native_nidx02a_encode_test.go | 89 +-
pkg/index/inverted/native_nidx02b_e2e_test.go | 148 --
pkg/index/inverted/native_nidx02b_fixture_test.go | 500 -------
pkg/index/inverted/native_nidx02b_plugin_test.go | 750 ----------
pkg/index/inverted/native_plugin.go | 1440 --------------------
.../native_plugin_contract_hardening_test.go | 198 ---
pkg/index/inverted/native_plugin_lazy_test.go | 442 ------
pkg/index/inverted/native_plugin_review_test.go | 303 ----
.../native_readonly_count_publication_e2e_test.go | 3 +-
.../native_readonly_doccount_generations_test.go | 2 +-
.../inverted/native_readonly_doccount_test.go | 2 +-
.../inverted/native_series_store_lazy_test.go | 91 --
.../native_sparse_docvalues_legacy_test.go | 65 -
pkg/index/inverted/native_writer_ownership_test.go | 64 -
pkg/index/inverted/nidx01d_review_test.go | 6 +-
pkg/index/inverted/query.go | 675 +--------
pkg/index/inverted/query_scan_budget_test.go | 86 --
pkg/index/inverted/repair_page.go | 170 ---
pkg/index/inverted/repair_page_tdd_test.go | 135 --
pkg/index/{inverted => metrics}/metrics.go | 95 +-
pkg/index/native/criteria/criteria.go | 249 ++++
pkg/index/native/criteria/criteria_test.go | 293 ++++
pkg/index/native/external.go | 135 +-
pkg/index/native/external_dedup_test.go | 234 ++++
pkg/index/native/identifier_docvalues_test.go | 117 ++
pkg/index/native/insert_if_absent_test.go | 492 +++++++
pkg/index/native/lifecycle.go | 2 +
pkg/index/native/native.go | 215 +++
pkg/index/native/native_test.go | 92 ++
pkg/index/native/owner.go | 502 ++++++-
pkg/index/native/owner_test.go | 4 +-
pkg/index/native/pattern_term_test.go | 293 ++++
pkg/index/native/presence_cache.go | 166 +++
pkg/index/native/query.go | 262 +++-
pkg/index/native/repair_test.go | 56 +
pkg/index/native/sort_cursor.go | 18 +-
pkg/index/nativeadapter/adapter.go | 2 +-
pkg/index/nativeadapter/store.go | 12 +-
pkg/index/nativeanalysis/analyzer.go | 22 +-
pkg/index/nativeanalysis/oracle_test.go | 24 +
.../stream/native_presence_integration_test.go | 2 +-
pkg/query/logical/tag_filter.go | 40 +-
pkg/query/model/model.go | 22 +-
pkg/query/vectorized/measure/plan/dispatch.go | 51 +-
.../vectorized/measure/plan/entity_extract.go | 236 ++++
.../vectorized/measure/plan/entity_extract_test.go | 413 ++++++
pkg/query/vectorized/measure/plan/scan.go | 12 +-
.../nidx03_rollback_service_traffic_insert.json | 44 +
.../nidx03_rollback_service_traffic_update.json | 16 +
.../nidx03/legacy_series_index_read.go.tmpl | 284 ++++
test/rollback/nidx03/previous_release.go | 229 ++++
test/rollback/nidx03/rollback_test.go | 317 +++++
test/rollback/nidx03/series_index_test.go | 95 ++
162 files changed, 12599 insertions(+), 9960 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index ff55fae7e..89318cd21 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -12,6 +12,7 @@ Release Notes.
- Serve the Stream element index, the dump index resolver, and `bydbctl analyze` from the native index: exact, existence, range, NOT, MATCH (keyword, simple, standard, and URL analyzers), sort, snapshot, and external-segment receive.
- [Breaking Change] Property no longer indexes tags named `source`. Schema properties carry their full JSON spec in that tag; the value is still stored, returned, and sortable, but a Property query filtering on a tag named `source` no longer matches.
- Route native index file operations through `pkg/fs`, which gains page-cache advice (`AdvisePageCache`, `SetCached`) and error-returning `MkdirAll`, `ReadDirLimit`, `Lstat`, and `SyncDir`.
+- [Breaking Change] Cut the per-segment series index (Measure, Stream, and Trace `seg-*/sidx`) over to the native index engine and remove the legacy bluge-backed series store: `pkg/index/inverted`'s series-only query builders, matchers, and tests are gone, and `banyand/internal/storage` no longer imports `pkg/index/inverted` or `github.com/blugelabs/bluge` at all (enforced by a dependency-guard test). There is no engine flag and no switch between engines; upgrades are whole-cluster (stop [...]
- Add logging related flags to the FODC proxy and agent. Every BanyanDB binary now shares the same logging flags and the matching `BYDB_LOGGING_*` environment variables.
- [Breaking Change] Remove the row-based query execution path from the Stream, Measure and Trace query engines. Measure TopN pre-aggregation is unchanged and still executes row-based. The `--stream-vectorized-enabled`, `--trace-vectorized-enabled` and `--measure-vectorized-enabled` flags stay registered but no longer select an engine; `=false` now fails fast at startup on standalone, data and liaison nodes. A query shape the vectorized engine cannot plan now returns an error instead of f [...]
diff --git a/banyand/backup/lifecycle/roundtrip_test.go b/banyand/backup/lifecycle/roundtrip_test.go
index 46fd1d0fd..b1eef7f86 100644
--- a/banyand/backup/lifecycle/roundtrip_test.go
+++ b/banyand/backup/lifecycle/roundtrip_test.go
@@ -60,7 +60,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/convert"
localfs "github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/node"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
@@ -1015,7 +1015,7 @@ func roundtripAllSidxDirsHaveSnapshot(groupRoot string) bool {
// exist (e.g. the live current-day segment while the data lands in the
// previous day around midnight); its series index holds no documents and
// therefore never gets a committed .snp, so requiring one would hang.
- if docs, _ := inverted.ReadOnlyDocCount(sidxDir); docs == 0 {
+ if docs, _ := native.ReadOnlyDocCount(sidxDir); docs == 0 {
continue
}
entries, readErr := os.ReadDir(sidxDir)
diff --git a/banyand/cmd/dump/property.go b/banyand/cmd/dump/property.go
index b82f9ae47..bfddccf3e 100644
--- a/banyand/cmd/dump/property.go
+++ b/banyand/cmd/dump/property.go
@@ -20,6 +20,7 @@ package main
import (
"context"
"encoding/csv"
+ "errors"
"fmt"
"os"
"sort"
@@ -33,13 +34,18 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
"github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
+// Property-document field names mirror the unexported layout
+// banyand/property/db writes (banyand/property/db/shard.go).
+const (
+ propertySourceField = "_source"
+ propertyDeleteField = "_deleted"
+)
+
type propertyDumpOptions struct {
shardPath string
criteriaJSON string
@@ -129,9 +135,8 @@ type propertyRowData struct {
}
type propertyDumpContext struct {
+ generation *native.ReadOnlyGeneration
tagFilter logical.TagFilter
- store index.SeriesStore
- seriesMap map[common.SeriesID]string
writer *csv.Writer
opts propertyDumpOptions
projectionTags []string
@@ -144,37 +149,31 @@ func newPropertyDumpContext(opts propertyDumpOptions) (*propertyDumpContext, err
opts: opts,
}
- // Open inverted index store
- l := logger.GetLogger("dump-property")
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: opts.shardPath,
- Logger: l,
- })
- if err != nil {
- return nil, fmt.Errorf("failed to open property shard: %w", err)
- }
- ctx.store = store
-
- // Load series map
- ctx.seriesMap, err = loadPropertySeriesMap(opts.shardPath)
+ // Open the shard's native index read-only: a read-only generation, never
+ // a writable owner, so no lock is acquired on a directory a live node may
+ // still have open.
+ generation, err := native.OpenReadOnlyGeneration(opts.shardPath)
if err != nil {
- fmt.Fprintf(os.Stderr, "Warning: Failed to load series information: %v\n", err)
- ctx.seriesMap = nil
- } else {
- fmt.Fprintf(os.Stderr, "Loaded %d series from shard\n", len(ctx.seriesMap))
+ if errors.Is(err, native.ErrNoSnapshot) {
+ fmt.Fprintf(os.Stderr, "Warning: shard has no committed data yet\n")
+ generation = nil
+ } else {
+ return nil, fmt.Errorf("failed to open property shard: %w", err)
+ }
}
+ ctx.generation = generation
// Parse criteria if provided
if opts.criteriaJSON != "" {
var criteria *modelv1.Criteria
criteria, err = parsePropertyCriteriaJSON(opts.criteriaJSON)
if err != nil {
- store.Close()
+ ctx.close()
return nil, fmt.Errorf("failed to parse criteria: %w", err)
}
ctx.tagFilter, err = logical.BuildSimpleTagFilter(criteria)
if err != nil {
- store.Close()
+ ctx.close()
return nil, fmt.Errorf("failed to build tag filter: %w", err)
}
fmt.Fprintf(os.Stderr, "Applied criteria filter\n")
@@ -191,7 +190,7 @@ func newPropertyDumpContext(opts propertyDumpOptions) (*propertyDumpContext, err
if len(ctx.projectionTags) > 0 {
ctx.tagColumns = ctx.projectionTags
} else {
- ctx.tagColumns, err = discoverPropertyTagColumns(ctx.store)
+ ctx.tagColumns, err = discoverPropertyTagColumns(ctx.generation)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: Failed to discover tag columns: %v\n", err)
ctx.tagColumns = []string{}
@@ -200,7 +199,7 @@ func newPropertyDumpContext(opts propertyDumpOptions) (*propertyDumpContext, err
}
if err := ctx.initOutput(); err != nil {
- store.Close()
+ ctx.close()
return nil, err
}
@@ -224,103 +223,109 @@ func (ctx *propertyDumpContext) initOutput() error {
}
func (ctx *propertyDumpContext) close() {
- if ctx.store != nil {
- ctx.store.Close()
+ if ctx.generation != nil {
+ _ = ctx.generation.Close()
}
if ctx.writer != nil {
ctx.writer.Flush()
}
}
-func (ctx *propertyDumpContext) processProperties() error {
- // Use SeriesIterator to iterate through all series and query properties for each
- searchCtx := context.Background()
- iter, err := ctx.store.SeriesIterator(searchCtx)
- if err != nil {
- return fmt.Errorf("failed to create series iterator: %w", err)
- }
- defer iter.Close()
-
- projection := []index.FieldKey{
- {TagName: "_id"},
- {TagName: "_timestamp"},
- {TagName: "_source"},
- {TagName: "_deleted"},
+// walkPropertyRows visits every live document in the shard's native index and
+// decodes it into a propertyRowData. It is factored out so tests can drive the
+// same decode path the dump tool uses.
+func walkPropertyRows(ctx context.Context, generation *native.ReadOnlyGeneration, visit func(propertyRowData) error) error {
+ if generation == nil {
+ return nil
}
-
- var allResults []index.SeriesDocument
- seriesCount := 0
-
- // Iterate through all series
- for iter.Next() {
- series := iter.Val()
- if len(series.EntityValues) == 0 {
- continue
- }
-
- seriesCount++
- // Build query for this specific series
- seriesMatchers := []index.SeriesMatcher{
- {
- Match: series.EntityValues,
- Type: index.SeriesMatcherTypeExact,
- },
- }
-
- iq, err := ctx.store.BuildQuery(seriesMatchers, nil, nil)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Warning: Failed to build query for series: %v\n", err)
- continue
- }
-
- // Search properties for this series
- results, err := ctx.store.Search(searchCtx, projection, iq, 10000)
+ return generation.VisitLiveDocuments(ctx, func(doc native.StoredDocument) error {
+ row, ok, err := decodePropertyRow(doc)
if err != nil {
- fmt.Fprintf(os.Stderr, "Warning: Failed to search properties for series: %v\n", err)
- continue
+ return err
}
-
- allResults = append(allResults, results...)
- }
-
- fmt.Fprintf(os.Stderr, "Found %d properties across %d series\n", len(allResults), seriesCount)
-
- // Process each result
- for _, result := range allResults {
- sourceBytes := result.Fields["_source"]
- if sourceBytes == nil {
- continue
+ if !ok {
+ return nil
}
+ return visit(row)
+ })
+}
- var property propertyv1.Property
- if err := protojson.Unmarshal(sourceBytes, &property); err != nil {
- fmt.Fprintf(os.Stderr, "Warning: Failed to unmarshal property: %v\n", err)
- continue
+// decodePropertyRow extracts a propertyRowData from one native stored
+// document. The identifier (_id) and timestamp (_timestamp) are written by
+// the native encoder for every document regardless of IdentifierDocValues
+// (NIDX-03 §2.2); _source and _deleted are the only other fields
+// banyand/property/db/shard.go stores (buildUpdateDocument). A document
+// without a _source value is not a property row -- ok is false -- so callers
+// never try to unmarshal an empty payload.
+//
+// A malformed _source or _timestamp is reported as a warning on stderr, not
+// a hard error: one corrupt row must not abort the rest of the dump, the
+// same tolerance the previous (bluge-backed) dump tool had. walkPropertyRows
+// still aborts on a genuine read-path failure (VisitStoredFields itself
+// erroring, which signals the underlying segment is unreadable, not that one
+// document's payload is malformed).
+func decodePropertyRow(doc native.StoredDocument) (propertyRowData, bool, error) {
+ var row propertyRowData
+ var sourceBytes []byte
+ visitErr := doc.VisitStoredFields(func(name string, value []byte) bool {
+ switch name {
+ case identifierFieldName:
+ row.id = append([]byte(nil), value...)
+ case timestampFieldName:
+ ts, decodeErr := native.DecodeTimestamp(value)
+ if decodeErr != nil {
+ fmt.Fprintf(os.Stderr, "warning: skipping _timestamp for document %x: %v\n", value, decodeErr)
+ return true
+ }
+ row.timestamp = ts
+ case propertySourceField:
+ sourceBytes = append([]byte(nil), value...)
+ case propertyDeleteField:
+ if len(value) > 0 {
+ row.deleteTime = convert.BytesToInt64(value)
+ }
}
+ return true
+ })
+ if visitErr != nil {
+ return propertyRowData{}, false, fmt.Errorf("visit stored fields: %w", visitErr)
+ }
+ if len(sourceBytes) == 0 {
+ return propertyRowData{}, false, nil
+ }
+ var property propertyv1.Property
+ if err := protojson.Unmarshal(sourceBytes, &property); err != nil {
+ fmt.Fprintf(os.Stderr, "warning: skipping document %x with malformed _source: %v\n", row.id, err)
+ return propertyRowData{}, false, nil
+ }
+ row.property = &property
+ if len(row.id) > 0 {
+ row.seriesID = common.SeriesID(convert.Hash(row.id))
+ }
+ return row, true, nil
+}
- var deleteTime int64
- if result.Fields["_deleted"] != nil {
- deleteTime = convert.BytesToInt64(result.Fields["_deleted"])
- }
+const (
+ identifierFieldName = "_id"
+ timestampFieldName = "_timestamp"
+)
- seriesID := common.SeriesID(0)
- if len(result.Key.EntityValues) > 0 {
- seriesID = common.SeriesID(convert.Hash(result.Key.EntityValues))
- }
+func (ctx *propertyDumpContext) processProperties() error {
+ searchCtx := context.Background()
+ var allResults []propertyRowData
+ if walkErr := walkPropertyRows(searchCtx, ctx.generation, func(row propertyRowData) error {
+ allResults = append(allResults, row)
+ return nil
+ }); walkErr != nil {
+ return fmt.Errorf("failed to walk property shard: %w", walkErr)
+ }
- row := propertyRowData{
- id: result.Key.EntityValues,
- property: &property,
- timestamp: result.Timestamp,
- deleteTime: deleteTime,
- seriesID: seriesID,
- }
+ fmt.Fprintf(os.Stderr, "Found %d properties\n", len(allResults))
- // Apply tag filter if specified
+ for _, row := range allResults {
if ctx.shouldSkip(row) {
continue
}
-
if err := ctx.writeRow(row); err != nil {
return err
}
@@ -355,11 +360,11 @@ func (ctx *propertyDumpContext) shouldSkip(row propertyRowData) bool {
func (ctx *propertyDumpContext) writeRow(row propertyRowData) error {
if ctx.opts.csvOutput {
- if err := writePropertyRowAsCSV(ctx.writer, row, ctx.tagColumns, ctx.seriesMap); err != nil {
+ if err := writePropertyRowAsCSV(ctx.writer, row, ctx.tagColumns); err != nil {
return err
}
} else {
- writePropertyRowAsText(row, ctx.rowNum+1, ctx.opts.verbose, ctx.projectionTags, ctx.seriesMap)
+ writePropertyRowAsText(row, ctx.rowNum+1, ctx.opts.verbose, ctx.projectionTags)
}
ctx.rowNum++
return nil
@@ -373,38 +378,6 @@ func (ctx *propertyDumpContext) printSummary() {
fmt.Printf("\nTotal rows: %d\n", ctx.rowNum)
}
-func loadPropertySeriesMap(shardPath string) (map[common.SeriesID]string, error) {
- l := logger.GetLogger("dump-property")
-
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: shardPath,
- Logger: l,
- })
- if err != nil {
- return nil, fmt.Errorf("failed to open series index: %w", err)
- }
- defer store.Close()
-
- ctx := context.Background()
- iter, err := store.SeriesIterator(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to create series iterator: %w", err)
- }
- defer iter.Close()
-
- seriesMap := make(map[common.SeriesID]string)
- for iter.Next() {
- series := iter.Val()
- if len(series.EntityValues) > 0 {
- seriesID := common.SeriesID(convert.Hash(series.EntityValues))
- seriesText := string(series.EntityValues)
- seriesMap[seriesID] = seriesText
- }
- }
-
- return seriesMap, nil
-}
-
func parsePropertyCriteriaJSON(criteriaJSON string) (*modelv1.Criteria, error) {
criteria := &modelv1.Criteria{}
err := protojson.Unmarshal([]byte(criteriaJSON), criteria)
@@ -430,43 +403,24 @@ func parsePropertyProjectionTags(projectionStr string) []string {
return result
}
-func discoverPropertyTagColumns(store index.SeriesStore) ([]string, error) {
- // Build query to get a sample property
- queryReq := &propertyv1.QueryRequest{
- Limit: 1,
- }
-
- iq, err := inverted.BuildPropertyQuery(queryReq, "_group", "_entity_id")
- if err != nil {
- return nil, fmt.Errorf("failed to build property query: %w", err)
- }
-
- projection := []index.FieldKey{
- {TagName: "_source"},
- }
-
- searchCtx := context.Background()
- results, err := store.Search(searchCtx, projection, iq, 1)
- if err != nil {
- return nil, fmt.Errorf("failed to search properties: %w", err)
- }
-
- if len(results) == 0 {
- return []string{}, nil
+// discoverPropertyTagColumns samples the first live property row to discover
+// the tag names CSV output should carry as columns.
+func discoverPropertyTagColumns(generation *native.ReadOnlyGeneration) ([]string, error) {
+ var sample *propertyv1.Property
+ errStop := errors.New("dump: stop walk")
+ walkErr := walkPropertyRows(context.Background(), generation, func(row propertyRowData) error {
+ sample = row.property
+ return errStop
+ })
+ if walkErr != nil && !errors.Is(walkErr, errStop) {
+ return nil, fmt.Errorf("failed to sample properties: %w", walkErr)
}
-
- sourceBytes := results[0].Fields["_source"]
- if sourceBytes == nil {
+ if sample == nil {
return []string{}, nil
}
- var property propertyv1.Property
- if err := protojson.Unmarshal(sourceBytes, &property); err != nil {
- return nil, fmt.Errorf("failed to unmarshal property: %w", err)
- }
-
tagNames := make(map[string]bool)
- for _, tag := range property.Tags {
+ for _, tag := range sample.Tags {
tagNames[tag.Key] = true
}
@@ -479,18 +433,12 @@ func discoverPropertyTagColumns(store index.SeriesStore) ([]string, error) {
return result, nil
}
-func writePropertyRowAsText(row propertyRowData, rowNum int, verbose bool, projectionTags []string, seriesMap map[common.SeriesID]string) {
+func writePropertyRowAsText(row propertyRowData, rowNum int, verbose bool, projectionTags []string) {
fmt.Printf("Row %d:\n", rowNum)
fmt.Printf(" ID: %s\n", string(row.id))
fmt.Printf(" Timestamp: %s\n", formatTimestamp(row.timestamp))
fmt.Printf(" SeriesID: %d\n", row.seriesID)
- if seriesMap != nil {
- if seriesText, ok := seriesMap[row.seriesID]; ok {
- fmt.Printf(" Series: %s\n", seriesText)
- }
- }
-
if row.property.Metadata != nil {
fmt.Printf(" Group: %s\n", row.property.Metadata.Group)
fmt.Printf(" Name: %s\n", row.property.Metadata.Name)
@@ -543,14 +491,7 @@ func writePropertyRowAsText(row propertyRowData, rowNum int, verbose bool, proje
fmt.Printf("\n")
}
-func writePropertyRowAsCSV(writer *csv.Writer, row propertyRowData, tagColumns []string, seriesMap map[common.SeriesID]string) error {
- seriesText := ""
- if seriesMap != nil {
- if text, ok := seriesMap[row.seriesID]; ok {
- seriesText = text
- }
- }
-
+func writePropertyRowAsCSV(writer *csv.Writer, row propertyRowData, tagColumns []string) error {
group := ""
name := ""
entityID := ""
@@ -571,7 +512,7 @@ func writePropertyRowAsCSV(writer *csv.Writer, row propertyRowData, tagColumns [
string(row.id),
formatTimestamp(row.timestamp),
fmt.Sprintf("%d", row.seriesID),
- seriesText,
+ string(row.id),
group,
name,
entityID,
diff --git a/banyand/cmd/dump/property_test.go b/banyand/cmd/dump/property_test.go
index 1eb6bba5c..3e1918350 100644
--- a/banyand/cmd/dump/property_test.go
+++ b/banyand/cmd/dump/property_test.go
@@ -19,24 +19,31 @@ package main
import (
"context"
+ "encoding/csv"
+ "path/filepath"
+ "strings"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "google.golang.org/protobuf/encoding/protojson"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
"github.com/apache/skywalking-banyandb/banyand/property/db"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/test"
)
-// TestDumpPropertyShardFormat tests that the dump tool can parse property shard data.
-// This test creates a real shard using the property module's operations,
-// then verifies the dump tool can correctly parse it.
+// TestDumpPropertyShardFormat tests that the dump tool's native read-only walk
+// (walkPropertyRows / decodePropertyRow, banyand/cmd/dump/property.go) can
+// parse property shard data written by the real native property store. This
+// creates a real shard using the property module's own write path, then
+// verifies the dump tool -- not a reimplementation of it -- decodes every row
+// correctly.
func TestDumpPropertyShardFormat(t *testing.T) {
tmpPath, defFn := test.Space(require.New(t))
defer defFn()
@@ -45,68 +52,24 @@ func TestDumpPropertyShardFormat(t *testing.T) {
shardPath, cleanup := createTestPropertyShardForDump(tmpPath)
defer cleanup()
- // Open the shard using dump tool functions
- l := logger.GetLogger("dump-property")
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: shardPath,
- Logger: l,
- })
+ generation, err := native.OpenReadOnlyGeneration(shardPath)
require.NoError(t, err, "should be able to open shard created by property module")
- defer store.Close()
-
- // Use SeriesIterator to iterate through all series and query properties
- searchCtx := context.Background()
- iter, err := store.SeriesIterator(searchCtx)
- require.NoError(t, err, "should be able to create series iterator")
- defer iter.Close()
-
- projection := []index.FieldKey{
- {TagName: "_id"},
- {TagName: "_timestamp"},
- {TagName: "_source"},
- {TagName: "_deleted"},
- }
-
- var allResults []index.SeriesDocument
- seriesCount := 0
-
- // Iterate through all series
- for iter.Next() {
- series := iter.Val()
- if len(series.EntityValues) == 0 {
- continue
- }
-
- seriesCount++
- // Build query for this specific series
- seriesMatchers := []index.SeriesMatcher{
- {
- Match: series.EntityValues,
- Type: index.SeriesMatcherTypeExact,
- },
- }
+ defer func() { _ = generation.Close() }()
- iq, err := store.BuildQuery(seriesMatchers, nil, nil)
- require.NoError(t, err, "should be able to build query for series")
-
- // Search properties for this series
- results, err := store.Search(searchCtx, projection, iq, 10000)
- require.NoError(t, err, "should be able to search properties for series")
-
- allResults = append(allResults, results...)
- }
+ var allResults []propertyRowData
+ walkErr := walkPropertyRows(context.Background(), generation, func(row propertyRowData) error {
+ allResults = append(allResults, row)
+ return nil
+ })
+ require.NoError(t, walkErr, "should be able to walk the shard's live property rows")
assert.Greater(t, len(allResults), 0, "should have at least one property")
- t.Logf("Found %d properties across %d series", len(allResults), seriesCount)
+ t.Logf("Found %d properties", len(allResults))
// Verify we can parse all properties
for i, result := range allResults {
- sourceBytes := result.Fields["_source"]
- require.NotNil(t, sourceBytes, "property %d should have source field", i)
-
- var prop propertyv1.Property
- err := protojson.Unmarshal(sourceBytes, &prop)
- require.NoError(t, err, "should be able to unmarshal property %d", i)
+ prop := result.property
+ require.NotNil(t, prop, "property %d should have a decoded source", i)
t.Logf("Property %d: Group=%s, Name=%s, EntityID=%s, ModRevision=%d, Tags=%d",
i, prop.Metadata.Group, prop.Metadata.Name, prop.Id, prop.Metadata.ModRevision, len(prop.Tags))
@@ -117,8 +80,10 @@ func TestDumpPropertyShardFormat(t *testing.T) {
assert.NotEmpty(t, prop.Metadata.Name, "property %d should have name", i)
assert.NotEmpty(t, prop.Id, "property %d should have entity ID", i)
- // Verify timestamp
- assert.Greater(t, result.Timestamp, int64(0), "property %d should have valid timestamp", i)
+ // Verify identity and timestamp were recovered from the native _id /
+ // _timestamp fields.
+ assert.NotEmpty(t, result.id, "property %d should have a decoded identifier", i)
+ assert.Greater(t, result.timestamp, int64(0), "property %d should have valid timestamp", i)
// Verify tags if present
for _, tag := range prop.Tags {
@@ -136,3 +101,136 @@ func createTestPropertyShardForDump(tmpPath string) (string, func()) {
fileSystem := fs.NewLocalFileSystem()
return db.CreateTestShardForDump(tmpPath, fileSystem)
}
+
+const dumpTestGroup = "dump-test-group"
+
+// createDeletedPropertyShard opens a fresh property database, writes one
+// live property and one property it then Deletes (a soft-delete tombstone:
+// the document survives with _deleted set, per banyand/property/db/shard.go
+// buildUpdateDocument/Delete), and returns the on-disk shard-0 path plus a
+// cleanup. R5's own ask: decodePropertyRow's _deleted handling and the CSV/
+// criteria paths had no test before this.
+func createDeletedPropertyShard(t *testing.T) (shardPath string, aliveID, deletedID []byte) {
+ t.Helper()
+ tmpPath, cleanupSpace := test.Space(require.New(t))
+ t.Cleanup(cleanupSpace)
+
+ database, err := db.OpenDB(context.Background(), db.Config{
+ Location: tmpPath,
+ MetricsScopeName: "dump_property_deleted_test",
+ FlushInterval: time.Second,
+ Index: db.IndexConfig{WaitForPersistence: true},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = database.Close() })
+
+ alive := newDumpTestProperty("alive-id", 1, "status", "ok")
+ require.NoError(t, database.Update(context.Background(), 0, db.GetPropertyID(alive), alive))
+ toDelete := newDumpTestProperty("deleted-id", 1, "status", "gone")
+ require.NoError(t, database.Update(context.Background(), 0, db.GetPropertyID(toDelete), toDelete))
+ require.NoError(t, database.Delete(context.Background(), [][]byte{db.GetPropertyID(toDelete)}, time.Now()))
+
+ return filepath.Join(tmpPath, dumpTestGroup, "shard-0"), db.GetPropertyID(alive), db.GetPropertyID(toDelete)
+}
+
+func newDumpTestProperty(id string, modRevision int64, tagKey, tagValue string) *propertyv1.Property {
+ return &propertyv1.Property{
+ Metadata: &commonv1.Metadata{Group: dumpTestGroup, Name: "dump-test-name", ModRevision: modRevision},
+ Id: id,
+ Tags: []*modelv1.Tag{
+ {Key: tagKey, Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: tagValue}}}},
+ },
+ }
+}
+
+// TestDumpPropertyDecodesDeletedFlag is R5's _deleted coverage:
+// decodePropertyRow must surface a soft-deleted property's deleteTime
+// (non-zero), and a live property's must stay zero.
+func TestDumpPropertyDecodesDeletedFlag(t *testing.T) {
+ shardPath, aliveID, deletedID := createDeletedPropertyShard(t)
+
+ generation, err := native.OpenReadOnlyGeneration(shardPath)
+ require.NoError(t, err)
+ defer func() { _ = generation.Close() }()
+
+ rows := map[string]propertyRowData{}
+ require.NoError(t, walkPropertyRows(context.Background(), generation, func(row propertyRowData) error {
+ rows[string(row.id)] = row
+ return nil
+ }))
+
+ alive, found := rows[string(aliveID)]
+ require.True(t, found, "the live property must be present")
+ assert.Equal(t, int64(0), alive.deleteTime, "a live property's deleteTime must be zero")
+
+ deleted, found := rows[string(deletedID)]
+ require.True(t, found, "the soft-deleted property must still be walked (it is a tombstone row, not absent)")
+ assert.Greater(t, deleted.deleteTime, int64(0), "a soft-deleted property's deleteTime must be set")
+}
+
+// TestDumpPropertyCSVOutput is R5's CSV coverage: writePropertyRowAsCSV
+// (the function newPropertyDumpContext's --csv path calls per row) must
+// produce a row whose ID/Group/Name/EntityID/Deleted columns match the
+// decoded propertyRowData, through the real csv.Writer encoding (quoting,
+// field order) rather than a hand-built string comparison.
+func TestDumpPropertyCSVOutput(t *testing.T) {
+ shardPath, aliveID, deletedID := createDeletedPropertyShard(t)
+
+ generation, err := native.OpenReadOnlyGeneration(shardPath)
+ require.NoError(t, err)
+ defer func() { _ = generation.Close() }()
+
+ rows := map[string]propertyRowData{}
+ require.NoError(t, walkPropertyRows(context.Background(), generation, func(row propertyRowData) error {
+ rows[string(row.id)] = row
+ return nil
+ }))
+
+ var buf strings.Builder
+ writer := csv.NewWriter(&buf)
+ require.NoError(t, writePropertyRowAsCSV(writer, rows[string(aliveID)], nil))
+ require.NoError(t, writePropertyRowAsCSV(writer, rows[string(deletedID)], nil))
+ writer.Flush()
+ require.NoError(t, writer.Error())
+
+ reader := csv.NewReader(strings.NewReader(buf.String()))
+ records, readErr := reader.ReadAll()
+ require.NoError(t, readErr)
+ require.Len(t, records, 2)
+
+ // Columns: ID, Timestamp, SeriesID, Series, Group, Name, EntityID, Deleted, ModRevision.
+ assert.Equal(t, string(aliveID), records[0][0])
+ assert.Equal(t, dumpTestGroup, records[0][4])
+ assert.Equal(t, "false", records[0][7], "the live property's CSV row must report Deleted=false")
+
+ assert.Equal(t, string(deletedID), records[1][0])
+ assert.Equal(t, "true", records[1][7], "the soft-deleted property's CSV row must report Deleted=true")
+}
+
+// TestDumpPropertyCriteriaFilter is R5's criteria-filter coverage:
+// propertyDumpContext.shouldSkip, built from the same --criteria JSON the
+// CLI flag accepts (parsePropertyCriteriaJSON + logical.BuildSimpleTagFilter,
+// newPropertyDumpContext's own construction path), must keep a property
+// whose tag matches and skip one that does not.
+func TestDumpPropertyCriteriaFilter(t *testing.T) {
+ shardPath, aliveID, deletedID := createDeletedPropertyShard(t)
+
+ ctx, err := newPropertyDumpContext(propertyDumpOptions{
+ shardPath: shardPath,
+ criteriaJSON: `{"condition":{"name":"status","op":"BINARY_OP_EQ","value":{"str":{"value":"ok"}}}}`,
+ })
+ require.NoError(t, err)
+ defer ctx.close()
+ require.NotNil(t, ctx.tagFilter, "a --criteria JSON string must build a non-nil tag filter")
+
+ var kept []string
+ require.NoError(t, walkPropertyRows(context.Background(), ctx.generation, func(row propertyRowData) error {
+ if !ctx.shouldSkip(row) {
+ kept = append(kept, string(row.id))
+ }
+ return nil
+ }))
+
+ assert.Contains(t, kept, string(aliveID), "the property matching status=ok must not be skipped")
+ assert.NotContains(t, kept, string(deletedID), "the property with status=gone must be skipped by the status=ok criteria")
+}
diff --git a/banyand/cmd/dump/sidx.go b/banyand/cmd/dump/sidx.go
index 50cd17f71..77fbcafeb 100644
--- a/banyand/cmd/dump/sidx.go
+++ b/banyand/cmd/dump/sidx.go
@@ -21,6 +21,7 @@ import (
"context"
"encoding/csv"
"encoding/json"
+ "errors"
"fmt"
"math"
"os"
@@ -41,8 +42,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
"github.com/apache/skywalking-banyandb/pkg/query/model"
@@ -372,38 +372,43 @@ func discoverPartIDs(segmentPath string) ([]uint64, error) {
}
func discoverSeriesIDs(segmentPath string) ([]common.SeriesID, error) {
- // Open the series index (sidx directory under segment)
+ // Open the series index (sidx directory under segment) read-only: a
+ // native read-only generation, never a writable owner, so no lock is
+ // acquired on a directory a live node may still have open.
seriesIndexPath := filepath.Join(segmentPath, "sidx")
- l := logger.GetLogger("dump-sidx")
-
- // Create inverted index store
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: seriesIndexPath,
- Logger: l,
- })
+ generation, err := native.OpenReadOnlyGeneration(seriesIndexPath)
if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return nil, nil
+ }
return nil, fmt.Errorf("failed to open series index: %w", err)
}
- defer store.Close()
-
- // Get series iterator
- ctx := context.Background()
- iter, err := store.SeriesIterator(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to create series iterator: %w", err)
- }
- defer iter.Close()
-
- // Collect all series IDs
+ defer func() { _ = generation.Close() }()
+
+ // Collect all series IDs. VisitIdentifiers walks term dictionaries
+ // per physical segment within the generation and deliberately includes
+ // terms from deleted documents (its own doc comment), so the same
+ // identifier -- and so the same computed SeriesID -- can surface more
+ // than once (e.g. an upsert that moved a series to a newer physical
+ // segment still leaves its term in the older one). Dedupe here so a
+ // series present in several physical segments is reported, and queried,
+ // exactly once.
+ seen := make(map[common.SeriesID]struct{})
var seriesIDs []common.SeriesID
- for iter.Next() {
- series := iter.Val()
+ visitErr := generation.VisitIdentifiers(context.Background(), func(entityValues []byte) bool {
// Compute series ID from EntityValues using hash
- if len(series.EntityValues) > 0 {
- seriesID := common.SeriesID(convert.Hash(series.EntityValues))
- seriesIDs = append(seriesIDs, seriesID)
+ if len(entityValues) > 0 {
+ sid := common.SeriesID(convert.Hash(entityValues))
+ if _, dup := seen[sid]; !dup {
+ seen[sid] = struct{}{}
+ seriesIDs = append(seriesIDs, sid)
+ }
}
+ return true
+ })
+ if visitErr != nil {
+ return nil, fmt.Errorf("failed to visit series identifiers: %w", visitErr)
}
return seriesIDs, nil
@@ -916,36 +921,27 @@ func parseProjectionTags(projectionStr string) []string {
func loadSeriesMap(segmentPath string) (map[common.SeriesID]string, error) {
seriesIndexPath := filepath.Join(segmentPath, "sidx")
- l := logger.GetLogger("dump-sidx")
-
- // Create inverted index store
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: seriesIndexPath,
- Logger: l,
- })
+ generation, err := native.OpenReadOnlyGeneration(seriesIndexPath)
if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return map[common.SeriesID]string{}, nil
+ }
return nil, fmt.Errorf("failed to open series index: %w", err)
}
- defer store.Close()
-
- // Get series iterator
- ctx := context.Background()
- iter, err := store.SeriesIterator(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to create series iterator: %w", err)
- }
- defer iter.Close()
+ defer func() { _ = generation.Close() }()
// Build map of SeriesID -> text representation
seriesMap := make(map[common.SeriesID]string)
- for iter.Next() {
- series := iter.Val()
- if len(series.EntityValues) > 0 {
- seriesID := common.SeriesID(convert.Hash(series.EntityValues))
+ visitErr := generation.VisitIdentifiers(context.Background(), func(entityValues []byte) bool {
+ if len(entityValues) > 0 {
+ seriesID := common.SeriesID(convert.Hash(entityValues))
// Convert EntityValues bytes to readable string
- seriesText := string(series.EntityValues)
- seriesMap[seriesID] = seriesText
+ seriesMap[seriesID] = string(entityValues)
}
+ return true
+ })
+ if visitErr != nil {
+ return nil, fmt.Errorf("failed to visit series identifiers: %w", visitErr)
}
return seriesMap, nil
diff --git a/banyand/cmd/dump/sidx_test.go b/banyand/cmd/dump/sidx_test.go
new file mode 100644
index 000000000..cee070ed1
--- /dev/null
+++ b/banyand/cmd/dump/sidx_test.go
@@ -0,0 +1,114 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+)
+
+// TestDiscoverSeriesIDs_DedupesAcrossPhysicalSegments is the L7 regression:
+// native.ReadOnlyGeneration.VisitIdentifiers walks term dictionaries per
+// physical segment, and (its own doc comment) "global lexical ordering
+// across segments is not promised" -- nothing collapses the same
+// identifier term across two physical segments into one visit. A
+// BatchInsertOnly admission (no existing-identifier check, unlike
+// BatchUpsert/BatchInsertIfAbsent) for the same identifier in two separate
+// batches reproduces this directly: both land as independent live
+// documents in their own physical segment, so the same computed SeriesID
+// surfaces twice. discoverSeriesIDs must report (and later query against)
+// each series exactly once, not once per physical-segment occurrence.
+func TestDiscoverSeriesIDs_DedupesAcrossPhysicalSegments(t *testing.T) {
+ root := t.TempDir()
+ segmentPath := filepath.Join(root, "seg-test")
+ sidxPath := filepath.Join(segmentPath, "sidx")
+ require.NoError(t, os.MkdirAll(segmentPath, 0o755))
+
+ lfs := fs.NewLocalFileSystem()
+ lockFile, err := lfs.CreateLockFile(filepath.Join(segmentPath, "lock"), 0o600)
+ require.NoError(t, err)
+ lease, err := native.NewFileRootLease(lockFile, segmentPath)
+ require.NoError(t, err)
+
+ owner, err := native.NewOwner(native.OwnerOptions{
+ Lease: lease, Path: sidxPath, IdentifierDocValues: true,
+ // Disable compaction so the two batches below stay as two separate
+ // physical segments instead of being merged away -- the scenario
+ // that leaves a superseded term behind in the older segment.
+ CompactionThreshold: -1,
+ })
+ require.NoError(t, err)
+
+ identifier := []byte("dup-series-identifier")
+ for i := range 2 {
+ done := make(chan error, 1)
+ batchErr := owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{
+ Identifier: identifier,
+ Timestamp: int64(i + 1),
+ }},
+ // BatchInsertOnly performs no existing-identifier check (unlike
+ // BatchUpsert/BatchInsertIfAbsent), so both calls admit the
+ // identifier as an independent live document in its own
+ // physical segment.
+ Mode: native.BatchInsertOnly,
+ PersistentCallback: func(err error) { done <- err },
+ })
+ require.NoError(t, batchErr)
+ require.NoError(t, <-done)
+ }
+ require.NoError(t, owner.Close())
+
+ seriesIDs, discoverErr := discoverSeriesIDs(segmentPath)
+ require.NoError(t, discoverErr)
+ require.Len(t, seriesIDs, 1, "the same identifier written twice across two physical segments must be reported once")
+ require.Equal(t, common.SeriesID(convert.Hash(identifier)), seriesIDs[0])
+}
+
+func TestDebugPrintLen(t *testing.T) {
+ root := t.TempDir()
+ segmentPath := filepath.Join(root, "seg-test")
+ sidxPath := filepath.Join(segmentPath, "sidx")
+ os.MkdirAll(segmentPath, 0o755)
+ lfs := fs.NewLocalFileSystem()
+ lockFile, _ := lfs.CreateLockFile(filepath.Join(segmentPath, "lock"), 0o600)
+ lease, _ := native.NewFileRootLease(lockFile, segmentPath)
+ owner, _ := native.NewOwner(native.OwnerOptions{Lease: lease, Path: sidxPath, IdentifierDocValues: true, CompactionThreshold: -1})
+ identifier := []byte("dup-series-identifier")
+ for i := 0; i < 2; i++ {
+ done := make(chan error, 1)
+ owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{Identifier: identifier, Timestamp: int64(i + 1)}},
+ Mode: native.BatchInsertOnly,
+ PersistentCallback: func(err error) { done <- err },
+ })
+ <-done
+ }
+ owner.Close()
+ ids, err := discoverSeriesIDs(segmentPath)
+ println("LEN:", len(ids), "ERR:", err == nil)
+}
diff --git a/banyand/internal/dump/helpers.go b/banyand/internal/dump/helpers.go
index 047af7bb5..bb6b7bb2c 100644
--- a/banyand/internal/dump/helpers.go
+++ b/banyand/internal/dump/helpers.go
@@ -19,6 +19,7 @@ package dump
import (
"context"
+ "errors"
"fmt"
"os"
"path/filepath"
@@ -29,14 +30,12 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
pkgdump "github.com/apache/skywalking-banyandb/pkg/dump"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
)
const (
dirNameSidx = "sidx"
dirNameMeta = "meta"
- logName = "dump"
)
// ReadData reads exactly len(buf) bytes from r at offset. Unlike fs.MustReadData
@@ -82,34 +81,31 @@ func DiscoverPartIDs(shardPath string) ([]uint64, error) {
return partIDs, nil
}
-// LoadSegmentSeriesMap loads the segment-level SeriesID -> EntityValues map from
-// the sidx inverted store under segmentPath. Requires a local filesystem path.
+// LoadSegmentSeriesMap loads the segment-level SeriesID -> EntityValues map
+// from the sidx series index under segmentPath, read-only (a native read-only
+// generation, never a writable owner -- no lock is acquired).
func LoadSegmentSeriesMap(segmentPath string) (map[common.SeriesID]string, error) {
seriesIndexPath := filepath.Join(segmentPath, dirNameSidx)
- store, err := inverted.NewStore(inverted.StoreOpts{
- Path: seriesIndexPath,
- Logger: logger.GetLogger(logName),
- })
+ generation, err := native.OpenReadOnlyGeneration(seriesIndexPath)
if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return map[common.SeriesID]string{}, nil
+ }
return nil, fmt.Errorf("failed to open series index: %w", err)
}
- defer store.Close()
-
- ctx := context.Background()
- iter, err := store.SeriesIterator(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to create series iterator: %w", err)
- }
- defer iter.Close()
+ defer func() { _ = generation.Close() }()
seriesMap := make(map[common.SeriesID]string)
- for iter.Next() {
- series := iter.Val()
- if len(series.EntityValues) > 0 {
- seriesID := common.SeriesID(convert.Hash(series.EntityValues))
- seriesMap[seriesID] = string(series.EntityValues)
+ visitErr := generation.VisitIdentifiers(context.Background(), func(entityValues []byte) bool {
+ if len(entityValues) > 0 {
+ seriesID := common.SeriesID(convert.Hash(entityValues))
+ seriesMap[seriesID] = string(entityValues)
}
+ return true
+ })
+ if visitErr != nil {
+ return nil, fmt.Errorf("failed to visit series identifiers: %w", visitErr)
}
return seriesMap, nil
diff --git a/banyand/internal/dump/measure/suite_test.go b/banyand/internal/dump/measure/suite_test.go
index 68ecbc23d..17fc8380a 100644
--- a/banyand/internal/dump/measure/suite_test.go
+++ b/banyand/internal/dump/measure/suite_test.go
@@ -47,7 +47,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
localfs "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -638,7 +638,7 @@ func findSidxSegmentPath(t *testing.T, root string) string {
count := 0
_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err == nil && d.IsDir() && d.Name() == "sidx" {
- c, e := inverted.ReadOnlyDocCount(p)
+ c, e := native.ReadOnlyDocCount(p)
candidates += fmt.Sprintf("{path=%s count=%d err=%v} ", p, c, e)
count++
if c > best {
diff --git a/banyand/internal/migration/fsutil.go b/banyand/internal/migration/fsutil.go
index ab8c3c22d..2bb367e26 100644
--- a/banyand/internal/migration/fsutil.go
+++ b/banyand/internal/migration/fsutil.go
@@ -25,9 +25,14 @@ import (
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
)
+// legacyLockFilename is the previous release's bluge exclusive-lock file
+// name (pkg/index/inverted.LockFilename), duplicated here so this package
+// never imports pkg/index/inverted for series-index work. A stale lock
+// traveling with a copied index dir would block the target from opening it.
+const legacyLockFilename = "bluge.pid"
+
// NoFsyncFS wraps a local FileSystem and skips per-file fsync /
// directory sync calls. Migration is a one-shot bulk import: a crash
// mid-run leaves the target root in an unusable state anyway and the
@@ -97,7 +102,7 @@ func CopyDir(src, dst string) (int64, error) {
return 0, err
}
for _, e := range entries {
- if !e.IsDir() && e.Name() == inverted.LockFilename {
+ if !e.IsDir() && e.Name() == legacyLockFilename {
continue
}
srcPath := filepath.Join(src, e.Name())
diff --git a/banyand/internal/migration/unionsidx.go b/banyand/internal/migration/unionsidx.go
index 7eecd7525..5f1487841 100644
--- a/banyand/internal/migration/unionsidx.go
+++ b/banyand/internal/migration/unionsidx.go
@@ -28,30 +28,39 @@ import (
"sync"
"sync/atomic"
- "github.com/blugelabs/bluge"
-
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ banyanfs "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/run"
)
-// The union sidx is rebuilt read-only via raw bluge — no banyandb store
-// lifecycle is needed.
+// lfs performs this builder's own lock-file management. It is stateless, so
+// one package-level instance suffices (matching pkg/index/native's own
+// package-level fileSystem).
+var lfs = banyanfs.NewLocalFileSystem()
+
+// The union sidx is rebuilt read-only via a native owner opened directly on
+// stagingPath — no banyandb TSDB/segment lifecycle is needed, just the
+// lock-backed root lease every native.Owner requires.
const (
- // Larger batches mean fewer bluge merger ticks; the merger loop
- // dominates CPU at small batch sizes. 50k keeps the merger work
- // off the hot path while still fitting in the per-process heap.
+ // Larger batches mean fewer owner admissions, which otherwise dominate
+ // CPU at small batch sizes. 50k keeps per-admission overhead off the hot
+ // path while still fitting in the per-process heap.
unionSidxBatchSize = 50000
segPrefix = "seg-"
sidxDirName = "sidx"
- // Stored bluge field names of sidx (inverted-index) documents, mirroring
- // the (unexported) layout written by pkg/index/inverted so this builder
- // can re-emit docs read via raw bluge.
+ // unionSidxLockFilename is the lock file this builder creates to back
+ // its own native.FileRootLease; it is removed once the build finishes.
+ unionSidxLockFilename = "lock"
+
+ // Stored field names of sidx (series-index) documents, mirroring the
+ // (unexported) reserved names pkg/index/native keeps private, so this
+ // builder can re-emit docs read via native.ReadOnlyGeneration.
sidxDocIDField = "_id"
sidxTimestampField = "_timestamp"
sidxVersionField = "_version"
@@ -76,15 +85,37 @@ func BuildGroupUnionSidx(ctx context.Context, srcGroupRoots []string, stagingPat
return "", fmt.Errorf("mkdir staging %q: %w", stagingPath, err)
}
- writer, err := bluge.OpenWriter(bluge.DefaultConfig(stagingPath))
+ // native.FileRootLease requires its owner's Path to be a proper
+ // subdirectory of the lease root (ValidatePath rejects root==Path), the
+ // same relationship a TSDB's lock (at the segment root) and its sidx
+ // subdirectory have. stagingPath is the directory this function's
+ // contract hands back as the series index itself (callers open it
+ // directly), so the lock lives one level up instead.
+ leaseRoot := filepath.Dir(stagingPath)
+ lockPath := filepath.Join(leaseRoot, unionSidxLockFilename)
+ lock, err := lfs.CreateLockFile(lockPath, storage.FilePerm)
+ if err != nil {
+ return "", fmt.Errorf("create union sidx lock %q: %w", lockPath, err)
+ }
+ defer func() {
+ _ = lock.Close()
+ _ = os.Remove(lockPath)
+ }()
+ lease, err := native.NewFileRootLease(lock, leaseRoot)
+ if err != nil {
+ return "", fmt.Errorf("create union sidx root lease: %w", err)
+ }
+ // NewOwner is a synchronous constructor and has no context-bearing API.
+ //nolint:contextcheck // construction does not perform cancellable I/O
+ owner, err := native.NewOwner(native.OwnerOptions{Lease: lease, Path: stagingPath, IdentifierDocValues: true})
if err != nil {
- return "", fmt.Errorf("open union sidx writer at %q: %w", stagingPath, err)
+ return "", fmt.Errorf("open union sidx owner at %q: %w", stagingPath, err)
}
closed := false
published := false
defer func() {
if !closed {
- _ = writer.Close()
+ _ = owner.Close()
}
if !published {
_ = os.RemoveAll(stagingPath)
@@ -145,7 +176,7 @@ func BuildGroupUnionSidx(ctx context.Context, srcGroupRoots []string, stagingPat
return
}
logf("union sidx: start scanning %s", srcSidxPath)
- count, scanned, mergeErr := mergeOneSourceSidxInto(taskCtx, srcSidxPath, writer, seen, &seenMu, &writerMu)
+ count, scanned, mergeErr := mergeOneSourceSidxInto(taskCtx, srcSidxPath, owner, seen, &seenMu, &writerMu)
if mergeErr != nil {
e := fmt.Errorf("merge %s: %w", srcSidxPath, mergeErr)
if firstErr.CompareAndSwap(nil, &e) {
@@ -183,12 +214,12 @@ dispatch:
}
inserted := int(insertedAtomic.Load())
scanned := int(scannedAtomic.Load())
- logf("union sidx: scan finished — scanned=%d uniqueSeries=%d dedup-skipped=%d; closing writer (final bluge merge)",
+ logf("union sidx: scan finished — scanned=%d uniqueSeries=%d dedup-skipped=%d; closing owner (final persist)",
scanned, inserted, scanned-inserted)
closed = true
- if closeErr := writer.Close(); closeErr != nil {
- return "", fmt.Errorf("close union sidx writer: %w", closeErr)
+ if closeErr := owner.Close(); closeErr != nil {
+ return "", fmt.Errorf("close union sidx owner: %w", closeErr)
}
if inserted == 0 {
return "", nil
@@ -228,63 +259,67 @@ func collectSourceSidxPaths(ctx context.Context, srcGroupRoots []string) ([]stri
func mergeOneSourceSidxInto(
ctx context.Context,
srcPath string,
- dst *bluge.Writer,
+ dst *native.Owner,
seen map[common.SeriesID]struct{},
seenMu *sync.Mutex,
writerMu *sync.Mutex,
) (inserted, scanned int, err error) {
- batch := bluge.NewBatch()
- batched := 0
+ generation, openErr := native.OpenReadOnlyGeneration(srcPath)
+ if openErr != nil {
+ if errors.Is(openErr, native.ErrNoSnapshot) {
+ return 0, 0, nil
+ }
+ return 0, 0, fmt.Errorf("open source %s: %w", srcPath, openErr)
+ }
+ defer func() { _ = generation.Close() }()
+
+ batch := make([]native.Document, 0, unionSidxBatchSize)
flush := func() error {
+ if len(batch) == 0 {
+ return nil
+ }
writerMu.Lock()
defer writerMu.Unlock()
- return dst.Batch(batch)
+ return dst.Batch(ctx, native.Batch{Documents: batch})
}
- walkErr := inverted.ReadOnlyWalkDocuments(ctx, srcPath, func(source inverted.StoredDocument) error {
+ walkErr := generation.VisitLiveDocuments(ctx, func(source native.StoredDocument) error {
scanned++
- doc, dup, buildErr := buildDocFromStoredDocumentLocked(source, seen, seenMu)
+ doc, dup, buildErr := buildNativeDocumentLocked(source, seen, seenMu)
if buildErr != nil {
return buildErr
}
if dup || doc == nil {
return nil
}
- batch.Insert(doc)
- batched++
+ batch = append(batch, *doc)
inserted++
- if batched >= unionSidxBatchSize {
+ if len(batch) >= unionSidxBatchSize {
if flushErr := flush(); flushErr != nil {
return fmt.Errorf("flush batch: %w", flushErr)
}
- batch = bluge.NewBatch()
- batched = 0
+ batch = make([]native.Document, 0, unionSidxBatchSize)
}
return nil
})
if walkErr != nil {
- if errors.Is(walkErr, inverted.ErrNoCommittedIndex) {
- return 0, 0, nil
- }
return inserted, scanned, fmt.Errorf("walk source %s: %w", srcPath, walkErr)
}
- if batched > 0 {
- if flushErr := flush(); flushErr != nil {
- return inserted, scanned, fmt.Errorf("flush tail batch: %w", flushErr)
- }
+ if flushErr := flush(); flushErr != nil {
+ return inserted, scanned, fmt.Errorf("flush tail batch: %w", flushErr)
}
return inserted, scanned, nil
}
-// buildDocFromStoredDocumentLocked rebuilds one series-index doc from its
-// stored fields, deduplicating by SeriesID under seenMu.
-func buildDocFromStoredDocumentLocked(
- source inverted.StoredDocument,
+// buildNativeDocumentLocked rebuilds one series-index doc from its stored
+// fields, deduplicating by SeriesID under seenMu.
+func buildNativeDocumentLocked(
+ source native.StoredDocument,
seen map[common.SeriesID]struct{},
seenMu *sync.Mutex,
-) (*bluge.Document, bool, error) {
+) (*native.Document, bool, error) {
var entityValues []byte
type storedField struct {
name string
@@ -321,30 +356,35 @@ func buildDocFromStoredDocumentLocked(
seen[series.ID] = struct{}{}
seenMu.Unlock()
- doc := bluge.NewDocument(string(entityValues))
+ doc := &native.Document{Identifier: entityValues}
for _, f := range fields {
switch f.name {
case sidxTimestampField:
- ts, decErr := bluge.DecodeDateTime(f.value)
+ ts, decErr := native.DecodeTimestamp(f.value)
if decErr != nil {
return nil, false, fmt.Errorf("decode timestamp on series %d: %w", series.ID, decErr)
}
- doc.AddField(bluge.NewDateTimeField(f.name, ts).StoreValue())
+ doc.Timestamp = ts
case sidxVersionField:
- doc.AddField(bluge.NewStoredOnlyField(f.name, f.value))
+ // _version is stored-only (storage.EncodeSeriesDocument never
+ // indexes it): Store must be set explicitly here, or the
+ // field carries neither an index entry nor a stored value and
+ // is silently dropped (NIDX-03 §6.1).
+ doc.Fields = append(doc.Fields, native.Field{Name: f.name, Value: f.value, Store: true})
default:
// Remaining stored fields are the series' entity-tag fields
// (e.g. "service"), written by the production series index as
- // keyword fields (pkg/index/inverted/inverted_series.go toDoc).
- // VisitStoredFields only returns the stored name+value, not the
- // original Index/NoSort/Analyzer flags, so they are re-emitted as
- // keyword+stored. This preserves exact-match (keyword) series
- // lookups — the only way the union sidx is queried for the
- // supported catalogs (entity tags are keyword, not analyzed) — but
- // would NOT preserve a non-keyword analyzer or doc-values sorting.
- // Such fields don't occur in the current series index; supporting
- // them would require threading the schema's field metadata in.
- doc.AddField(bluge.NewKeywordFieldBytes(f.name, f.value).StoreValue())
+ // indexed, stored, sortable keyword fields
+ // (storage.EncodeSeriesDocument). VisitStoredFields only returns
+ // the stored name+value, not the original Index/NoSort/Analyzer
+ // flags, so they are re-emitted the same way: indexed, stored and
+ // sortable, with no analyzer. This preserves exact-match series
+ // lookups -- the only way the union sidx is queried for the
+ // supported catalogs (entity tags are keyword, not analyzed) --
+ // but would NOT preserve a non-keyword analyzer. Such fields
+ // don't occur in the current series index; supporting them would
+ // require threading the schema's field metadata in.
+ doc.Fields = append(doc.Fields, native.Field{Name: f.name, Value: f.value, Index: true, Store: true, Sort: true})
}
}
return doc, false, nil
diff --git a/banyand/internal/migration/unionsidx_native_source_test.go b/banyand/internal/migration/unionsidx_native_source_test.go
index 5d3bc6698..4e5e40deb 100644
--- a/banyand/internal/migration/unionsidx_native_source_test.go
+++ b/banyand/internal/migration/unionsidx_native_source_test.go
@@ -36,14 +36,19 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
const (
// nativeVisitorPackage and nativeVisitorFunc name the BanyanDB-owned
- // read-only document visitor issue #14010 introduces.
- nativeVisitorPackage = "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- nativeVisitorFunc = "ReadOnlyWalkDocuments"
+ // read-only document visitor the series index cutover (NIDX-03) moved
+ // this package onto: a native.ReadOnlyGeneration opened directly,
+ // instead of the pkg/index/inverted wrapper issue #14010 introduced
+ // (itself already nativeice-backed, but still a pkg/index/inverted
+ // dependency the series-index cutover retires).
+ nativeVisitorPackage = "github.com/apache/skywalking-banyandb/pkg/index/native"
+ nativeVisitorFunc = "OpenReadOnlyGeneration"
// retiredReaderEntryPoint is the call that opens a retired third-party
// index reader directly on a source directory. This package reads sources
@@ -55,11 +60,8 @@ const (
// file shorter than it cannot carry a valid one.
iceFooterLength = 60
- // unionSourceTagName is the stored tag the seeded sources carry, and
- // publishedSeriesLimit bounds a destination lookup so a stray duplicate is
- // reported rather than silently truncated away.
- unionSourceTagName = "service"
- publishedSeriesLimit = 100
+ // unionSourceTagName is the stored tag the seeded sources carry.
+ unionSourceTagName = "service"
)
// TestSeriesUnionSourceReadReachesNativeVisitor is the structural half of the
@@ -211,9 +213,10 @@ func seedUnionSourceSidx(t *testing.T, groupRoot, segName string, seriesSet []un
return sidxDir
}
-// openPublishedDestination opens a published union index through the retained
-// series-store surface, which this milestone does not touch, so what it reports
-// cannot agree with a faulty source read by construction.
+// openPublishedDestination opens a published union index through the legacy
+// compatibility store, an independent reader from the native visitor the union
+// reads its sources with, so what it reports cannot agree with a faulty source
+// read by construction.
func openPublishedDestination(t *testing.T, sidxDir string) index.SeriesStore {
t.Helper()
store, err := inverted.NewStore(inverted.StoreOpts{Path: sidxDir, BatchWaitSec: 0})
@@ -223,23 +226,17 @@ func openPublishedDestination(t *testing.T, sidxDir string) index.SeriesStore {
// publishedUnionSeries looks one identity up in a published union index and
// reports the stored tag values it holds, or reports that the index holds no
-// live document under that identity. The lookup is an exact one so that the
-// answer for one series can never be contaminated by another's.
+// live document under that identity. The lookup is an exact identifier one so
+// that the answer for one series can never be contaminated by another's.
func publishedUnionSeries(t *testing.T, store index.SeriesStore, identity []byte) ([]string, bool) {
t.Helper()
- query, err := store.BuildQuery([]index.SeriesMatcher{
- {Type: index.SeriesMatcherTypeExact, Match: identity},
- }, nil, nil)
+ fields, err := store.StoredFields(context.Background(), identity, index.FieldKey{TagName: unionSourceTagName})
require.NoError(t, err)
- documents, err := store.Search(context.Background(),
- []index.FieldKey{{TagName: unionSourceTagName}}, query, publishedSeriesLimit)
- require.NoError(t, err)
- if len(documents) == 0 {
+ if fields == nil {
return nil, false
}
- require.Len(t, documents, 1, "identity %q must resolve to at most one live published document", identity)
- var values []string
- if value, stored := documents[0].Fields[unionSourceTagName]; stored && value != nil {
+ values := make([]string, 0, len(fields[unionSourceTagName]))
+ for _, value := range fields[unionSourceTagName] {
values = append(values, string(value))
}
sort.Strings(values)
@@ -266,7 +263,7 @@ func truncateUnionSourceSegment(t *testing.T, sidxDir string) {
// exists for: a lifecycle migration unions the series indexes of several node
// replicas into one staged index before broadcasting it. The union reads every
// source through the native document visitor and re-emits the survivors through
-// its retained destination writer, which this milestone does not change.
+// a native destination owner.
//
// Requirements proved here:
//
@@ -295,7 +292,7 @@ func TestE2ESeriesUnionNativeSource(t *testing.T) {
tester.NoError(err)
tester.Equal(staging, published, "a union that read live series must publish its staged index")
- publishedCount, countErr := inverted.ReadOnlyDocCount(published)
+ publishedCount, countErr := native.ReadOnlyDocCount(published)
tester.NoError(countErr)
tester.Equal(int64(2), publishedCount,
"the union must publish exactly the two live series, each once, even though the first is read twice")
@@ -322,7 +319,7 @@ func TestE2ESeriesUnionNativeSource(t *testing.T) {
corruptPublished, corruptErr := BuildGroupUnionSidx(context.Background(),
[]string{corruptRoot}, corruptStaging, nil)
- tester.ErrorIs(corruptErr, inverted.ErrCorruptIndex,
+ tester.ErrorIs(corruptErr, native.ErrCorrupt,
"a damaged source must be reported as a corrupt index, not as an opaque read failure")
tester.Empty(corruptPublished, "a union that failed on a damaged source must publish no staged index")
}
diff --git a/banyand/internal/migration/unionsidx_test.go b/banyand/internal/migration/unionsidx_test.go
index 33d3a5849..572812f58 100644
--- a/banyand/internal/migration/unionsidx_test.go
+++ b/banyand/internal/migration/unionsidx_test.go
@@ -31,6 +31,8 @@ import (
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -244,6 +246,70 @@ func TestBuildGroupUnionSidx_DeduplicatesAcrossSegments(t *testing.T) {
}
}
+// TestBuildGroupUnionSidx_PreservesVersionField is the L6 regression:
+// buildNativeDocumentLocked built a native.Field{Name, Value} for
+// sidxVersionField with no Store set, so the Index=false, Store=false
+// zero value carried neither an index entry nor a stored value -- _version
+// was silently dropped from the union sidx. It must survive the build,
+// matching storage.EncodeSeriesDocument's own "_version" is stored-only"
+// mapping.
+func TestBuildGroupUnionSidx_PreservesVersionField(t *testing.T) {
+ srcGroupRoot := t.TempDir()
+ stagingPath := filepath.Join(t.TempDir(), "union", "sidx")
+
+ seg := filepath.Join(srcGroupRoot, segPrefix+"20260101")
+ doc, sid := makeSidxSourceDoc(t, "alpha", "svc-a")
+ doc.AddField(bluge.NewKeywordFieldBytes(sidxVersionField, convert.Int64ToBytes(7)).StoreValue())
+ writeSidxAt(t, seg, []*bluge.Document{doc})
+
+ resultPath, err := BuildGroupUnionSidx(context.Background(), []string{srcGroupRoot}, stagingPath, nil)
+ if err != nil {
+ t.Fatalf("BuildGroupUnionSidx: %v", err)
+ }
+
+ generation, err := native.OpenReadOnlyGeneration(resultPath)
+ if err != nil {
+ t.Fatalf("OpenReadOnlyGeneration: %v", err)
+ }
+ defer func() { _ = generation.Close() }()
+
+ var gotVersion []byte
+ var found bool
+ visitErr := generation.VisitLiveDocuments(context.Background(), func(doc native.StoredDocument) error {
+ var entity []byte
+ var version []byte
+ _ = doc.VisitStoredFields(func(field string, value []byte) bool {
+ switch field {
+ case sidxDocIDField:
+ entity = append([]byte(nil), value...)
+ case sidxVersionField:
+ version = append([]byte(nil), value...)
+ }
+ return true
+ })
+ var s pbv1.Series
+ if len(entity) > 0 {
+ if err := s.Unmarshal(entity); err == nil && s.ID == sid {
+ found = true
+ gotVersion = version
+ }
+ }
+ return nil
+ })
+ if visitErr != nil {
+ t.Fatalf("VisitLiveDocuments: %v", visitErr)
+ }
+ if !found {
+ t.Fatalf("expected to find series %d in the union sidx", sid)
+ }
+ if len(gotVersion) == 0 {
+ t.Fatalf("expected a stored _version value, got none")
+ }
+ if got := convert.BytesToInt64(gotVersion); got != 7 {
+ t.Fatalf("expected _version 7, got %d", got)
+ }
+}
+
func TestBuildGroupUnionSidx_EmptyGroupYieldsEmptyPath(t *testing.T) {
srcGroupRoot := t.TempDir()
stagingPath := filepath.Join(t.TempDir(), "union", "sidx")
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index 5b9f17737..bbb1c7383 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -18,23 +18,52 @@
package storage
import (
+ "bytes"
"context"
- "maps"
+ "fmt"
+ "os"
"path"
+ "path/filepath"
+ "time"
"github.com/pkg/errors"
"go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ idxmetrics "github.com/apache/skywalking-banyandb/pkg/index/metrics"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ "github.com/apache/skywalking-banyandb/pkg/index/native/criteria"
+ "github.com/apache/skywalking-banyandb/pkg/index/nativeanalysis"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-const seriesIndexDirName = "sidx"
+const (
+ seriesIndexDirName = "sidx"
+
+ // identifierFieldName, timestampFieldName and versionFieldName mirror the
+ // reserved field names pkg/index/native keeps private; the series index
+ // needs its own copies to name them in TermSetRequest.Field, SortRequest.
+ // Field and ProjectHit's requested field list.
+ identifierFieldName = "_id"
+ timestampFieldName = "_timestamp"
+ versionFieldName = "_version"
+
+ // legacyLockFilename and legacyExternalSegmentTempDirName are the
+ // previous release's on-disk artifact names (pkg/index/inverted.LockFilename
+ // and pkg/index/inverted.ExternalSegmentTempDirName), duplicated here so
+ // the series index never imports pkg/index/inverted for series-index work.
+ // newSeriesIndex removes both on open (NIDX-03 §6.3): a rolled-back node's
+ // bluge writer lock and staging directory are meaningless once a native
+ // owner is opened on the same sidx directory.
+ legacyLockFilename = "bluge.pid"
+ legacyExternalSegmentTempDirName = "external-segment-temp"
+)
func (s *segment[T, O]) IndexDB() IndexDB {
// Snapshot s.index into a local: the field can be flipped to nil by a
@@ -54,72 +83,210 @@ func (s *segment[T, O]) Lookup(ctx context.Context, series []*pbv1.Series) (pbv1
if idx == nil {
return nil, ErrSegmentClosed
}
- sl, err := idx.filter(ctx, series, nil, nil, nil)
+ sl, err := idx.filter(ctx, series)
return sl.SeriesList, err
}
-// seriesIndex is the per-segment series index. Every method on this type
-// starts with `if s == nil { ... }` purely to absorb the typed-nil leak
-// that `segment.IndexDB()` could expose if a caller bypassed the producer
-// guard there. The guards intentionally do NOT make the type fully
-// nil-safe -- they assume that on a non-nil receiver, store / l / metrics
-// / p are all populated by newSeriesIndex (the only constructor).
+// seriesIndex is the per-segment series index. It is an application-layer
+// adapter over pkg/index/native, the same way the Property store
+// (banyand/property/db/native_property.go) is: the engine owns segments,
+// admission, persistence, sorting and projection; this type owns only the
+// series document mapping (EncodeSeriesDocument), identity matching
+// (seriesUniverse / subjectUniverse) and the IndexDB surface.
+//
+// Every method on this type starts with `if s == nil { ... }` purely to
+// absorb the typed-nil leak that `segment.IndexDB()` could expose if a
+// caller bypassed the producer guard there. The guards intentionally do NOT
+// make the type fully nil-safe -- they assume that on a non-nil receiver,
+// owner / l / metrics / p are all populated by newSeriesIndex (the only
+// constructor).
type seriesIndex struct {
- store index.SeriesStore
+ owner *native.Owner
l *logger.Logger
- metrics *inverted.Metrics
+ metrics *idxmetrics.Metrics
p common.Position
+ // wait selects synchronous persistence (SeriesIndexFlushTimeoutSeconds ==
+ // 0): Insert/Update blocks for the owner's durability callback before
+ // returning. When false, PersistInterval coalesces a write burst in
+ // memory before it is written, like the legacy engine's persister nap,
+ // and Insert/Update returns as soon as the batch is admitted.
+ wait bool
}
func newSeriesIndex(ctx context.Context, root string, flushTimeoutSeconds int64, cacheMaxBytes int,
- metrics *inverted.Metrics,
+ metrics *idxmetrics.Metrics, lease RootLease,
) (*seriesIndex, error) {
si := &seriesIndex{
l: logger.Fetch(ctx, "series_index"),
p: common.GetPosition(ctx),
}
- opts := inverted.StoreOpts{
- Path: path.Join(root, seriesIndexDirName),
- Logger: si.l,
- BatchWaitSec: flushTimeoutSeconds,
- CacheMaxBytes: cacheMaxBytes,
- EnableDeduplication: true,
- ExternalSegmentTempDir: path.Join(root, inverted.ExternalSegmentTempDirName),
- }
if metrics != nil {
- opts.Metrics = metrics
- si.metrics = opts.Metrics
+ si.metrics = metrics
}
- var err error
- if si.store, err = inverted.NewStore(opts); err != nil {
+ indexPath := path.Join(root, seriesIndexDirName)
+ removeLegacySeriesIndexArtifacts(si.l, root, indexPath)
+
+ si.wait = flushTimeoutSeconds <= 0
+ var persistInterval time.Duration
+ if !si.wait {
+ persistInterval = time.Duration(flushTimeoutSeconds) * time.Second
+ }
+ // NewOwner is a synchronous constructor and has no context-bearing API.
+ //nolint:contextcheck // construction does not perform cancellable I/O
+ owner, err := native.NewOwner(native.OwnerOptions{
+ Lease: lease,
+ Path: indexPath,
+ ExternalDedup: native.ExternalDedupKeepExisting,
+ PresenceCacheBytes: cacheMaxBytes,
+ IdentifierDocValues: true,
+ PersistInterval: persistInterval,
+ })
+ if err != nil {
return nil, err
}
+ si.owner = owner
return si, nil
}
+// removeLegacySeriesIndexArtifacts best-effort removes the previous
+// release's bluge exclusive-lock file (indexPath/bluge.pid, inside the sidx
+// directory itself) and external-segment staging directory
+// (root/external-segment-temp, a SIBLING of sidx -- the previous release's
+// inverted.StoreOpts.ExternalSegmentTempDir was rooted at the segment
+// directory, not inside Path). Both are meaningless to (and would otherwise
+// linger forever under) a native owner opened on the same directory. Errors
+// are logged, not panicked or returned: a stray legacy artifact that cannot
+// be removed (for example a permissions issue) must not block the segment
+// from opening.
+func removeLegacySeriesIndexArtifacts(l *logger.Logger, root, indexPath string) {
+ legacyLockPath := filepath.Join(indexPath, legacyLockFilename)
+ if _, statErr := os.Stat(legacyLockPath); statErr == nil {
+ if err := lfs.DeleteFile(legacyLockPath); err != nil {
+ l.Warn().Err(err).Str("path", legacyLockPath).Msg("failed to remove legacy series index lock file")
+ }
+ }
+ legacyExternalSegmentTempDir := filepath.Join(root, legacyExternalSegmentTempDirName)
+ if err := os.RemoveAll(legacyExternalSegmentTempDir); err != nil {
+ l.Warn().Err(err).Str("path", legacyExternalSegmentTempDir).
+ Msg("failed to remove legacy external-segment staging directory")
+ }
+}
+
+// EncodeSeriesDocument maps an index.Document -- the series-index write
+// contract every product (Measure, Stream, Trace) uses -- onto the native
+// engine's physical Document, per NIDX-03 §6.1:
+//
+// - EntityValues becomes Identifier (_id); a document with none is rejected.
+// - Timestamp maps straight to the engine's dedicated Timestamp field,
+// which the encoder indexes, stores and gives doc values only when it is
+// positive.
+// - Version, when positive, becomes a stored-only, unindexed "_version"
+// field (convert.Int64ToBytes); zero/negative carries no field at all.
+// - Each index.Field becomes a native.Field: Index=true fields are indexed
+// (with analyzer terms from pkg/index/nativeanalysis attached whenever
+// the field declares a non-keyword analyzer) and sorted unless NoSort;
+// Index=false fields are stored-only.
+//
+// It is exported for the offline tools that build sidx directories outside a
+// live seriesIndex: the union-sidx builder and the index-mode Measure copy.
+func EncodeSeriesDocument(doc index.Document) (native.Document, error) {
+ if len(doc.EntityValues) == 0 {
+ return native.Document{}, fmt.Errorf("series document has no entity values: %w", native.ErrInvalidDocument)
+ }
+ nd := native.Document{Identifier: bytes.Clone(doc.EntityValues), Timestamp: doc.Timestamp}
+ nd.Fields = make([]native.Field, 0, len(doc.Fields)+1)
+ for i := range doc.Fields {
+ f := &doc.Fields[i]
+ name := f.Key.Marshal()
+ term, ok := f.GetTerm().(*index.BytesTermValue)
+ if !ok {
+ return native.Document{}, fmt.Errorf("series field %q has unsupported term %T", name, f.GetTerm())
+ }
+ value := bytes.Clone(term.Value)
+ // An unindexed field is always stored, matching the previous
+ // release's bluge.NewStoredOnlyField (NIDX-03 §6.1): Store=false on
+ // an Index=false field would carry no field at all, silently
+ // dropping data callers expect SortedValue/ProjectHit to return.
+ store := f.Store || !f.Index
+ nf := native.Field{Name: name, Value: value, Store: store, Index: f.Index, Sort: f.Index && !f.NoSort}
+ if f.Index && f.Key.Analyzer != index.AnalyzerUnspecified && f.Key.Analyzer != index.AnalyzerKeyword {
+ terms, analyzeErr := nativeanalysis.Analyze(f.Key.Analyzer, value)
+ if analyzeErr != nil {
+ return native.Document{}, fmt.Errorf("analyze series field %q: %w", name, analyzeErr)
+ }
+ nf.Terms = make([]native.Term, 0, len(terms))
+ for _, t := range terms {
+ nf.Terms = append(nf.Terms, native.Term{Value: bytes.Clone(t.Value), Frequency: t.Frequency})
+ }
+ }
+ nd.Fields = append(nd.Fields, nf)
+ }
+ if doc.Version > 0 {
+ nd.Fields = append(nd.Fields, native.Field{Name: versionFieldName, Value: convert.Int64ToBytes(doc.Version), Store: true})
+ }
+ return nd, nil
+}
+
func (s *seriesIndex) Insert(docs index.Documents) error {
if s == nil {
return ErrSegmentClosed
}
- return s.store.InsertSeriesBatch(index.Batch{
- Documents: docs,
- })
+ return s.batch(docs, native.BatchInsertIfAbsent)
}
func (s *seriesIndex) Update(docs index.Documents) error {
if s == nil {
return ErrSegmentClosed
}
- return s.store.UpdateSeriesBatch(index.Batch{
- Documents: docs,
- })
+ return s.batch(docs, native.BatchUpsert)
+}
+
+// batch encodes docs and admits them through the owner under mode, blocking
+// for the durability callback when s.wait (SeriesIndexFlushTimeoutSeconds ==
+// 0; see newSeriesIndex).
+//
+// When !s.wait (the production default, PersistInterval > 0), no
+// PersistentCallback is registered: the owner only records a background
+// persistence failure into the error Close() surfaces when a flush has no
+// pending callbacks (drainPersistence's len(callbacks) == 0 branch). A
+// callback that nobody drains -- which is what an unconditional callback
+// would be here, since the async caller has already returned -- would
+// silently swallow that failure instead, leaving admitted series
+// unrecoverable with no error anywhere (see closeResourcesLocked / NIDX-03
+// §8/§11, "marks the segment failed").
+//
+// Batch is a synchronous constructor-adjacent admission call and has no
+// cancellable context of its own on the IndexDB surface.
+//
+//nolint:contextcheck // see above.
+func (s *seriesIndex) batch(docs index.Documents, mode native.BatchMode) error {
+ if len(docs) == 0 {
+ return nil
+ }
+ nativeDocs := make([]native.Document, len(docs))
+ for i := range docs {
+ nd, err := EncodeSeriesDocument(docs[i])
+ if err != nil {
+ return fmt.Errorf("encode series document %d: %w", i, err)
+ }
+ nativeDocs[i] = nd
+ }
+ if !s.wait {
+ return s.owner.Batch(context.Background(), native.Batch{Documents: nativeDocs, Mode: mode})
+ }
+ done := make(chan error, 1)
+ callback := func(err error) { done <- err }
+ if err := s.owner.Batch(context.Background(), native.Batch{Documents: nativeDocs, Mode: mode, PersistentCallback: callback}); err != nil {
+ return err
+ }
+ return <-done
}
func (s *seriesIndex) EnableExternalSegments() (index.ExternalSegmentStreamer, error) {
if s == nil {
return nil, ErrSegmentClosed
}
- return s.store.EnableExternalSegments()
+ return s.owner.EnableExternalSegments()
}
// Stats degrades to (0, 0) on a nil receiver as defense-in-depth in case
@@ -128,64 +295,37 @@ func (s *seriesIndex) Stats() (dataCount int64, dataSizeBytes int64) {
if s == nil {
return 0, 0
}
- return s.store.Stats()
+ return s.owner.Stats()
}
-func (s *seriesIndex) filter(ctx context.Context, series []*pbv1.Series,
- projection []index.FieldKey, secondaryQuery index.Query, timeRange *timestamp.TimeRange,
-) (data SeriesData, err error) {
+// ResetCache drops the series index's insert-presence cache. It never
+// touches admitted data (pkg/index/native.Owner.ResetPresenceCache).
+func (s *seriesIndex) ResetCache() {
if s == nil {
- return SeriesData{}, ErrSegmentClosed
- }
- if len(series) == 0 && secondaryQuery == nil && timeRange == nil {
- return data, nil
- }
- var seriesMatchers []index.SeriesMatcher
- if len(series) > 0 {
- seriesMatchers = make([]index.SeriesMatcher, len(series))
- for i := range series {
- seriesMatchers[i], err = convertEntityValuesToSeriesMatcher(series[i])
- if err != nil {
- return SeriesData{}, err
- }
- }
+ return
}
+ s.owner.ResetPresenceCache()
+}
- indexQuery, err := s.store.BuildQuery(seriesMatchers, secondaryQuery, timeRange)
- if err != nil {
- return SeriesData{}, err
- }
- tracer := query.GetTracer(ctx)
- if tracer != nil {
- span, _ := tracer.StartSpan(ctx, "seriesIndex.search")
- span.Tagf("query", "%s", indexQuery.String())
- defer func() {
- span.Tagf("matched", "%d", len(data.SeriesList))
- if len(data.Fields) > 0 {
- span.Tagf("field_length", "%d", len(data.Fields[0]))
- }
- if err != nil {
- span.Error(err)
- }
- span.Stop()
- }()
- }
- ss, err := s.store.Search(ctx, projection, indexQuery, 0)
- if err != nil {
- return SeriesData{}, err
- }
- if len(ss) == 0 {
- return SeriesData{}, nil
- }
- data.SeriesList, data.Fields, data.Timestamps, data.Versions, err = convertIndexSeriesToSeriesList(ss, len(projection) > 0)
- if err != nil {
- return SeriesData{}, errors.WithMessagef(err, "failed to convert index series to series list, matchers: %v, matched: %d", seriesMatchers, len(ss))
+// TakeFileSnapshot streams a consistent copy of the committed generation,
+// including admitted segments whose asynchronous persistence has not yet
+// completed, to destination.
+func (s *seriesIndex) TakeFileSnapshot(destination string) error {
+ if s == nil {
+ return ErrSegmentClosed
}
- return data, nil
+ return s.owner.TakeFileSnapshot(destination)
}
var emptySeriesMatcher = index.SeriesMatcher{}
+// convertEntityValuesToSeriesMatcher classifies one series template into an
+// exact, prefix or wildcard matcher over _id, exactly as the previous
+// release did: a template with no pbv1.AnyTagValue entries matches exactly;
+// one whose AnyTagValue run is only a tail matches as a prefix of the
+// marshaled leading values; and one with any AnyTagValue elsewhere (or more
+// than one run) matches as a wildcard, escaping to the same legacy pattern
+// language pkg/index/native's Wildcard dictionary expansion expects.
func convertEntityValuesToSeriesMatcher(series *pbv1.Series) (index.SeriesMatcher, error) {
var hasAny, hasWildcard bool
var prefixIndex int
@@ -239,190 +379,299 @@ func convertEntityValuesToSeriesMatcher(series *pbv1.Series) (index.SeriesMatche
}, nil
}
-func convertIndexSeriesToSeriesList(indexSeries []index.SeriesDocument, hasFields bool) (pbv1.SeriesList, FieldResultList, []int64, []int64, error) {
- seriesList := make(pbv1.SeriesList, 0, len(indexSeries))
- var fields FieldResultList
- if hasFields {
- fields = make(FieldResultList, 0, len(indexSeries))
- }
- var timestamps, versions []int64
- for _, s := range indexSeries {
- var series pbv1.Series
- if err := series.Unmarshal(s.Key.EntityValues); err != nil {
- return nil, nil, nil, nil, errors.WithMessagef(err, "failed to unmarshal series: %s", s.Key.EntityValues)
- }
- seriesList = append(seriesList, &series)
- if fields != nil {
- fields = append(fields, s.Fields)
+// toNativeTimeRange converts the shared timestamp.TimeRange into the native
+// engine's QueryScope.TimeRange shape. A nil input keeps the universe
+// unbounded in time.
+func toNativeTimeRange(tr *timestamp.TimeRange) *native.TimeRange {
+ if tr == nil {
+ return nil
+ }
+ return &native.TimeRange{
+ Lower: tr.Start.UnixNano(), Upper: tr.End.UnixNano(),
+ IncludesLower: tr.IncludeStart, IncludesUpper: tr.IncludeEnd,
+ }
+}
+
+// seriesUniverse builds the _id MatchTermsSet universe (NIDX-03 §6.2 step 1,
+// "with series"): one request carrying the exact terms, prefixes and
+// wildcard patterns every series matcher contributes, ORed together exactly
+// as the previous release's boolean should-query did.
+func seriesUniverse(ctx context.Context, view *native.ReadView, series []*pbv1.Series, timeRange *timestamp.TimeRange,
+) ([]native.QueryHit, []index.SeriesMatcher, error) {
+ matchers := make([]index.SeriesMatcher, len(series))
+ var exact, prefix, wildcard [][]byte
+ for i := range series {
+ matcher, err := convertEntityValuesToSeriesMatcher(series[i])
+ if err != nil {
+ return nil, nil, err
}
- if s.Timestamp > 0 {
- timestamps = append(timestamps, s.Timestamp)
+ matchers[i] = matcher
+ switch matcher.Type {
+ case index.SeriesMatcherTypeExact:
+ exact = append(exact, matcher.Match)
+ case index.SeriesMatcherTypePrefix:
+ prefix = append(prefix, matcher.Match)
+ case index.SeriesMatcherTypeWildcard:
+ wildcard = append(wildcard, matcher.Match)
+ default:
+ return nil, nil, errors.Errorf("unsupported series matcher type: %v", matcher.Type)
}
- if s.Version > 0 {
- versions = append(versions, s.Version)
+ }
+ hits, err := view.MatchTermsSet(ctx, native.TermSetRequest{
+ Field: identifierFieldName, Terms: exact, Prefix: prefix, Wildcard: wildcard,
+ Mode: native.MatchAnyTerm, MaxTerms: ^uint64(0),
+ Scope: native.QueryScope{TimeRange: toNativeTimeRange(timeRange)},
+ })
+ return hits, matchers, err
+}
+
+// subjectUniverse builds the _im_name MatchTermsSet universe (NIDX-03 §6.2
+// step 1, "without series"): index-mode Measure reads scope their universe to
+// one literal term, the measure's name, instead of series matchers. An empty
+// subject deliberately yields an empty (not unbounded) universe.
+func subjectUniverse(ctx context.Context, view *native.ReadView, subject string, timeRange *timestamp.TimeRange) ([]native.QueryHit, error) {
+ if subject == "" {
+ return nil, nil
+ }
+ return view.MatchTermsSet(ctx, native.TermSetRequest{
+ Field: index.IndexModeName, Terms: [][]byte{convert.StringToBytes(subject)},
+ Mode: native.MatchAnyTerm, MaxTerms: ^uint64(0),
+ Scope: native.QueryScope{TimeRange: toNativeTimeRange(timeRange)},
+ })
+}
+
+// orderFieldName resolves the engine field SortHits/ProjectSortValue sort on
+// for a given index.OrderBy, matching the previous release's SeriesSort.
+// order.Index is required for OrderByTypeIndex; a nil one is a caller bug,
+// not a corrupt-but-tolerable state, so it is reported as an error here
+// rather than dereferenced, matching the previous release's own tolerance of
+// the combination (SeriesSort was never reached for it: the gate in search
+// below only calls into this function when order.Index != nil).
+func orderFieldName(order *index.OrderBy) (string, error) {
+ switch order.Type {
+ case index.OrderByTypeTime:
+ return timestampFieldName, nil
+ case index.OrderByTypeIndex:
+ if order.Index == nil {
+ return "", errors.New("order by index requires a non-nil index rule")
}
+ fk := index.FieldKey{IndexRuleID: order.Index.Metadata.Id}
+ return fk.Marshal(), nil
+ default:
+ return "", errors.Errorf("unsupported order by type: %v", order.Type)
}
- return seriesList, fields, timestamps, versions, nil
}
-func (s *seriesIndex) Search(ctx context.Context, series []*pbv1.Series, opts IndexSearchOpts,
+// search is the shared engine-call sequence behind Search, SearchWithoutSeries
+// and the legacy-named filter helper Lookup uses (NIDX-03 §6.2):
+//
+// 1. Universe: seriesUniverse (series matchers) when series is non-empty,
+// otherwise subjectUniverse (index mode). The universe is always this
+// explicit MatchTermsSet result, even when it is empty, so sorting an
+// empty candidate set below never panics.
+// 2. Filter: pkg/index/native/criteria.Filter over opts.Criteria.
+// 3. Order: SortHits on the index-rule field only for an index order
+// (opts.Order.Index != nil); a time order or no order at all keeps the
+// hits in MatchTermsSet's order and leaves sortedValues nil, matching
+// the previous release (a sorted index-mode/measure search changes
+// banyand/measure/query.go's needsSorting decision).
+// 4. Results: ProjectHit for the timestamp, "_version" and the projection,
+// series from the identifier, and ProjectSortValue when sorted. The
+// unsorted path also charges the query memory budget per the previous
+// release's parseResult/readSeriesDocument; the sorted path never did.
+func (s *seriesIndex) search(ctx context.Context, series []*pbv1.Series, subject string, opts IndexSearchOpts,
) (sd SeriesData, sortedValues [][]byte, err error) {
if s == nil {
return SeriesData{}, nil, ErrSegmentClosed
}
tracer := query.GetTracer(ctx)
+ var span *query.Span
if tracer != nil {
- var span *query.Span
- span, ctx = tracer.StartSpan(ctx, "seriesIndex.Search")
- if opts.Query != nil {
- span.Tagf("secondary_query", "%s", opts.Query.String())
- }
+ span, ctx = tracer.StartSpan(ctx, "seriesIndex.search")
defer func() {
if err != nil {
span.Error(err)
}
+ span.Tagf("matched", "%d", len(sd.SeriesList))
span.Stop()
}()
}
- if opts.Order == nil || opts.Order.Index == nil {
- sd, err = s.filter(ctx, series, opts.Projection, opts.Query, opts.TimeRange)
- if err != nil {
- return sd, nil, err
- }
- return sd, nil, nil
- }
- var span *query.Span
- if tracer != nil {
- span, _ = tracer.StartSpan(ctx, "sort")
- span.Tagf("preload", "%d", opts.PreloadSize)
- defer func() {
- if err != nil {
- span.Error(err)
- }
- span.Stop()
- }()
+ view, acquireErr := s.owner.Acquire(ctx)
+ if acquireErr != nil {
+ return SeriesData{}, nil, acquireErr
}
- seriesMatchers := make([]index.SeriesMatcher, len(series))
- for i := range series {
- seriesMatchers[i], err = convertEntityValuesToSeriesMatcher(series[i])
- if err != nil {
- return sd, nil, err
- }
+ defer func() {
+ err = multierr.Append(err, view.Close())
+ }()
+
+ var universe []native.QueryHit
+ if len(series) > 0 {
+ universe, _, err = seriesUniverse(ctx, view, series, opts.TimeRange)
+ } else {
+ universe, err = subjectUniverse(ctx, view, subject, opts.TimeRange)
}
- query, err := s.store.BuildQuery(seriesMatchers, opts.Query, opts.TimeRange)
if err != nil {
- return sd, nil, err
+ return SeriesData{}, nil, err
}
- iter, err := s.store.SeriesSort(ctx, query, opts.Order,
- opts.PreloadSize, opts.Projection)
- if err != nil {
- return sd, nil, err
+
+ hits, filterErr := criteria.Filter(ctx, view, universe, opts.Criteria, opts.Fields)
+ if filterErr != nil {
+ return SeriesData{}, nil, errors.WithMessage(filterErr, "filter series index")
}
- defer func() {
- err = multierr.Append(err, iter.Close())
- }()
- var r int
- for iter.Next() {
- r++
- val := iter.Val()
- var series pbv1.Series
- if err = series.Unmarshal(val.EntityValues); err != nil {
- return sd, nil, errors.WithMessagef(err, "failed to unmarshal series: %s", val.EntityValues)
+ // sorted selects SortHits/ProjectSortValue only for an index order
+ // (opts.Order.Index != nil), matching the previous release and NIDX-03
+ // §6.2: a time order (resolveOrderBy's {Type: OrderByTypeTime, Index:
+ // nil}) takes the unsorted path below, keeping hits in MatchTermsSet
+ // order and sortedValues nil. Sorting on every non-nil Order
+ // (including time order) would change index-mode output order and flip
+ // banyand/measure/query.go's needsSorting decision.
+ sorted := opts.Order != nil && opts.Order.Index != nil
+ var sortField string
+ if sorted {
+ sortField, err = orderFieldName(opts.Order)
+ if err != nil {
+ return SeriesData{}, nil, err
}
- sd.SeriesList = append(sd.SeriesList, &series)
- sd.Timestamps = append(sd.Timestamps, val.Timestamp)
- sd.Versions = append(sd.Versions, val.Version)
- if len(opts.Projection) > 0 {
- sd.Fields = append(sd.Fields, maps.Clone(val.Values))
+ hits, err = view.SortHits(ctx, hits, native.SortRequest{Field: sortField, Desc: opts.Order.Sort == modelv1.Sort_SORT_DESC})
+ if err != nil {
+ return SeriesData{}, nil, err
+ }
+ sortedValues = make([][]byte, 0, len(hits))
+ }
+
+ // Query memory budget: charged only on the unsorted path, matching the
+ // previous release's parseResult/readSeriesDocument (the sorted path's
+ // SeriesSort/sortIterator never charged). This one-time charge accounts
+ // for parseResult's fixed per-call overhead plus the requested
+ // projection; per-document and per-field-value charges follow below.
+ if !sorted {
+ if chargeErr := query.Charge(ctx, 1024+uint64(len(opts.Projection))*64); chargeErr != nil {
+ return SeriesData{}, nil, chargeErr
}
- sortedValues = append(sortedValues, val.SortedValue)
}
- if span != nil {
- span.Tagf("query", "%s", iter.Query().String())
- span.Tagf("rounds", "%d", r)
- span.Tagf("size", "%d", len(sd.SeriesList))
+
+ fields := make([]string, 0, len(opts.Projection)+1)
+ for i := range opts.Projection {
+ fields = append(fields, opts.Projection[i].Marshal())
}
- return sd, sortedValues, err
-}
+ fields = append(fields, versionFieldName)
+ hasProjection := len(opts.Projection) > 0
-func (s *seriesIndex) SearchWithoutSeries(ctx context.Context, opts IndexSearchOpts) (sd SeriesData, sortedValues [][]byte, err error) {
- if s == nil {
- return SeriesData{}, nil, ErrSegmentClosed
+ sd.SeriesList = make(pbv1.SeriesList, 0, len(hits))
+ sd.Timestamps = make([]int64, 0, len(hits))
+ sd.Versions = make([]int64, 0, len(hits))
+ sd.TimestampSet = make([]bool, 0, len(hits))
+ sd.VersionSet = make([]bool, 0, len(hits))
+ if hasProjection {
+ sd.Fields = make(FieldResultList, 0, len(hits))
}
- tracer := query.GetTracer(ctx)
- if tracer != nil {
- var span *query.Span
- span, ctx = tracer.StartSpan(ctx, "seriesIndex.SearchWithoutSeries")
- if opts.Query != nil {
- span.Tagf("secondary_query", "%s", opts.Query.String())
+
+ for _, hit := range hits {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return SeriesData{}, nil, ctxErr
+ }
+ if appendErr := appendSearchHit(ctx, view, hit, fields, opts, sorted, sortField, hasProjection, &sd, &sortedValues); appendErr != nil {
+ return SeriesData{}, nil, appendErr
}
- defer func() {
- if err != nil {
- span.Error(err)
- }
- span.Stop()
- }()
}
+ return sd, sortedValues, nil
+}
- if opts.Order == nil || opts.Order.Index == nil {
- sd, err = s.filter(ctx, nil, opts.Projection, opts.Query, opts.TimeRange)
- if err != nil {
- return sd, nil, err
+// appendSearchHit extends sd (and sortedValues, on the sorted path) with one
+// hit's projected row: step 4 of search's doc comment, factored out to keep
+// search itself within the project's complexity budget. It charges the
+// query memory budget per hit and per projected field value on the unsorted
+// path only, matching the previous release's readSeriesDocument.
+func appendSearchHit(ctx context.Context, view *native.ReadView, hit native.QueryHit, fields []string,
+ opts IndexSearchOpts, sorted bool, sortField string, hasProjection bool, sd *SeriesData, sortedValues *[][]byte,
+) error {
+ projected, projectErr := view.ProjectHit(ctx, hit, fields...)
+ if projectErr != nil {
+ return projectErr
+ }
+ if !sorted {
+ if chargeErr := query.ChargeResult(ctx, 256+uint64(len(opts.Projection))*64); chargeErr != nil {
+ return chargeErr
+ }
+ if chargeErr := query.Charge(ctx, uint64(len(projected.Identifier))); chargeErr != nil {
+ return chargeErr
}
- return sd, nil, nil
}
- var span *query.Span
- if tracer != nil {
- span, _ = tracer.StartSpan(ctx, "sort")
- span.Tagf("preload", "%d", opts.PreloadSize)
- defer func() {
- if err != nil {
- span.Error(err)
+ var ser pbv1.Series
+ if unmarshalErr := ser.Unmarshal(projected.Identifier); unmarshalErr != nil {
+ return errors.WithMessagef(unmarshalErr, "failed to unmarshal series: %s", projected.Identifier)
+ }
+ sd.SeriesList = append(sd.SeriesList, &ser)
+
+ hasTimestamp := projected.Timestamp > 0
+ sd.Timestamps = append(sd.Timestamps, projected.Timestamp)
+ sd.TimestampSet = append(sd.TimestampSet, hasTimestamp)
+
+ var version int64
+ versionValues := projected.Fields[versionFieldName]
+ hasVersion := len(versionValues) > 0
+ if hasVersion {
+ // Last stored value wins, matching the previous release's
+ // visit-and-overwrite projection semantics.
+ version = convert.BytesToInt64(versionValues[len(versionValues)-1])
+ }
+ sd.Versions = append(sd.Versions, version)
+ sd.VersionSet = append(sd.VersionSet, hasVersion)
+
+ if hasProjection {
+ row := make(map[string][]byte, len(opts.Projection))
+ for i := range opts.Projection {
+ name := opts.Projection[i].Marshal()
+ values := projected.Fields[name]
+ if len(values) == 0 {
+ continue
}
- span.Stop()
- }()
+ value := values[len(values)-1]
+ if !sorted {
+ if chargeErr := query.Charge(ctx, uint64(len(value))); chargeErr != nil {
+ return chargeErr
+ }
+ }
+ row[name] = value
+ }
+ sd.Fields = append(sd.Fields, row)
}
- sortQuery := opts.Query
- if opts.TimeRange != nil {
- sortQuery, err = s.store.BuildQuery(nil, opts.Query, opts.TimeRange)
- if err != nil {
- return sd, nil, err
+ if sorted {
+ sortValue, _, sortErr := view.ProjectSortValue(ctx, hit, sortField)
+ if sortErr != nil {
+ return sortErr
}
+ *sortedValues = append(*sortedValues, sortValue)
}
- iter, err := s.store.SeriesSort(ctx, sortQuery, opts.Order,
- opts.PreloadSize, opts.Projection)
- if err != nil {
- return sd, nil, err
+ return nil
+}
+
+func (s *seriesIndex) Search(ctx context.Context, series []*pbv1.Series, opts IndexSearchOpts,
+) (SeriesData, [][]byte, error) {
+ if s == nil {
+ return SeriesData{}, nil, ErrSegmentClosed
}
- defer func() {
- err = multierr.Append(err, iter.Close())
- }()
+ return s.search(ctx, series, "", opts)
+}
- var r int
- for iter.Next() {
- r++
- val := iter.Val()
- var series pbv1.Series
- if err = series.Unmarshal(val.EntityValues); err != nil {
- return sd, nil, errors.WithMessagef(err, "failed to unmarshal series: %s", val.EntityValues)
- }
- sd.SeriesList = append(sd.SeriesList, &series)
- sd.Timestamps = append(sd.Timestamps, val.Timestamp)
- sd.Versions = append(sd.Versions, val.Version)
- if len(opts.Projection) > 0 {
- sd.Fields = append(sd.Fields, maps.Clone(val.Values))
- }
- sortedValues = append(sortedValues, val.SortedValue)
+func (s *seriesIndex) SearchWithoutSeries(ctx context.Context, opts IndexSearchOpts) (SeriesData, [][]byte, error) {
+ if s == nil {
+ return SeriesData{}, nil, ErrSegmentClosed
}
- if span != nil {
- span.Tagf("query", "%s", iter.Query().String())
- span.Tagf("rounds", "%d", r)
- span.Tagf("size", "%d", len(sd.SeriesList))
+ return s.search(ctx, nil, opts.IndexModeSubject, opts)
+}
+
+// filter is segment.Lookup's narrow entry point: series matchers only, no
+// criteria, no projection, no order, no index-mode subject, no time range.
+func (s *seriesIndex) filter(ctx context.Context, series []*pbv1.Series) (SeriesData, error) {
+ if s == nil {
+ return SeriesData{}, ErrSegmentClosed
}
- return sd, sortedValues, err
+ sd, _, err := s.search(ctx, series, "", IndexSearchOpts{})
+ return sd, err
}
func (s *seriesIndex) Close() error {
@@ -430,5 +679,5 @@ func (s *seriesIndex) Close() error {
return nil
}
s.metrics.DeleteAll(s.p.SegLabelValues()...)
- return s.store.Close()
+ return s.owner.Close()
}
diff --git a/banyand/internal/storage/index_async_persist_failure_test.go b/banyand/internal/storage/index_async_persist_failure_test.go
new file mode 100644
index 000000000..586a1e7a9
--- /dev/null
+++ b/banyand/internal/storage/index_async_persist_failure_test.go
@@ -0,0 +1,113 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// buildOneSeriesDoc builds a single, minimal index.Document for Insert.
+func buildOneSeriesDoc(t *testing.T, subject string) index.Documents {
+ t.Helper()
+ var series pbv1.Series
+ series.Subject = subject
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "s00"}}}}
+ require.NoError(t, series.Marshal())
+ return index.Documents{{EntityValues: append([]byte(nil), series.Buffer...), Timestamp: 1}}
+}
+
+// TestSeriesIndex_Insert_AsyncPersistFailureSurfacesOnClose is the L1
+// regression. seriesIndex.batch used to always register a
+// PersistentCallback, even in production's async mode
+// (SeriesIndexFlushTimeoutSeconds > 0, s.wait == false). The owner only
+// records a background persistence failure into the error Owner.Close()
+// returns when a flush has NO pending callbacks (drainPersistence's
+// len(callbacks) == 0 branch); an unconditional callback that nobody drains
+// in the async case (Insert already returned) silently swallowed the
+// failure instead, so admitted series were lost with no error anywhere and
+// closeResourcesLocked never saw the failure.
+//
+// This drives the real production path -- newSeriesIndex's async mode and
+// seriesIndex.Insert -- rather than calling owner.Batch directly (the
+// adjacent TestCloseResourcesLocked_SeriesIndexCloseErrorDoesNotPanic's own
+// comment admits it bypasses batch() this way, which is exactly why it
+// could not have caught this bug). The failure is induced by replacing the
+// sidx directory with a plain file after construction: admission is purely
+// in-memory and still succeeds, but the later persist attempt (the async
+// PersistInterval flush, or Close's own final flush) fails to recreate
+// "sidx" as a directory ("not a directory").
+func TestSeriesIndex_Insert_AsyncPersistFailureSurfacesOnClose(t *testing.T) {
+ ctx := context.Background()
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "error"}))
+ dir, fn := setUp(require.New(t))
+ defer fn()
+
+ // flushTimeoutSeconds > 0 selects async mode (s.wait == false), the
+ // production default whenever SeriesIndexFlushTimeoutSeconds is
+ // configured.
+ si, err := newSeriesIndex(ctx, dir, 10, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ indexPath := filepath.Join(dir, seriesIndexDirName)
+ require.NoError(t, os.RemoveAll(indexPath))
+ require.NoError(t, os.WriteFile(indexPath, []byte("not a directory"), 0o600))
+
+ require.NoError(t, si.Insert(buildOneSeriesDoc(t, testSubjectSvc)), "async admission is in-memory only and must still succeed")
+
+ closeErr := si.Close()
+ require.Error(t, closeErr, "a background persistence failure must surface through Close, not vanish silently")
+}
+
+// TestSegment_AsyncPersistFailureViaInsertMarksSegmentFailed is the
+// segment-level half of the same L1 regression: closeResourcesLocked must
+// observe the same Close() error (and so set CloseFailed) when it was
+// produced through seriesIndex.Insert's real async admission path, not only
+// through a raw owner.Batch call.
+func TestSegment_AsyncPersistFailureViaInsertMarksSegmentFailed(t *testing.T) {
+ ctx := context.Background()
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "error"}))
+ dir, fn := setUp(require.New(t))
+ defer fn()
+
+ si, err := newSeriesIndex(ctx, dir, 10, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ indexPath := filepath.Join(dir, seriesIndexDirName)
+ require.NoError(t, os.RemoveAll(indexPath))
+ require.NoError(t, os.WriteFile(indexPath, []byte("not a directory"), 0o600))
+ require.NoError(t, si.Insert(buildOneSeriesDoc(t, testSubjectSvc)))
+
+ seg := &segment[*MockTSTable, any]{index: si, l: logger.GetLogger("test")}
+ require.NotPanics(t, func() {
+ seg.mu.Lock()
+ seg.closeResourcesLocked()
+ seg.mu.Unlock()
+ })
+ require.True(t, seg.CloseFailed(), "an async persist failure reached through Insert must mark the segment failed")
+ require.Nil(t, seg.index)
+}
diff --git a/banyand/internal/storage/index_crash_cut_test.go b/banyand/internal/storage/index_crash_cut_test.go
new file mode 100644
index 000000000..04d674953
--- /dev/null
+++ b/banyand/internal/storage/index_crash_cut_test.go
@@ -0,0 +1,150 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// TestSeriesIndex_CrashCut_AfterBatch_ReopenedRootIsComplete pins NIDX-03
+// §12 item 6. The first batch is admitted and this test waits for it to
+// become durable (a real on-disk generation); a second batch is then
+// admitted and the owner is ABANDONED without ever calling Close -- no
+// graceful shutdown, no final drain -- the same way a crashed process never
+// runs one. Reopening the SAME directory with the real production
+// constructor (newSeriesIndex, a fresh lease -- a crashed process's lease
+// is gone with it) must see a complete generation (never a torn count
+// between the two batch sizes) and the data durable before the cut must
+// have survived.
+func TestSeriesIndex_CrashCut_AfterBatch_ReopenedRootIsComplete(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 10, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ const firstBatch = 5
+ require.NoError(t, si.Insert(buildTestSeriesDocsNamed(t, "crash-cut-before", firstBatch)))
+
+ indexPath := filepath.Join(dir, seriesIndexDirName)
+ require.Eventually(t, func() bool {
+ generation, openErr := native.OpenReadOnlyGeneration(indexPath)
+ if openErr != nil {
+ require.True(t, errors.Is(openErr, native.ErrNoSnapshot), "unexpected open error: %v", openErr)
+ return false
+ }
+ defer func() { _ = generation.Close() }()
+ count, countErr := generation.VisibleDocCount()
+ require.NoError(t, countErr)
+ return count == firstBatch
+ }, 5*time.Second, time.Millisecond, "the first batch must become durable before the cut")
+
+ // Admit a second batch and abandon si right here -- no Close, no final
+ // drain -- simulating a crash in the window between admission and the
+ // next scheduled asynchronous persist. si is deliberately never closed
+ // again; its in-memory lease is simply discarded, the same way a real
+ // crash releases a file-backed lease without any explicit Revoke/Close.
+ const secondBatch = 3
+ require.NoError(t, si.Insert(buildTestSeriesDocsNamed(t, "crash-cut-after", secondBatch)))
+
+ si2, err := newSeriesIndex(ctx, dir, 10, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si2.Close()) }()
+
+ generation, err := native.OpenReadOnlyGeneration(indexPath)
+ require.NoError(t, err, "a directory abandoned without Close must still reopen as a complete generation")
+ defer func() { _ = generation.Close() }()
+ count, err := generation.VisibleDocCount()
+ require.NoError(t, err)
+ require.True(t, count == firstBatch || count == firstBatch+secondBatch,
+ "reopened generation must be complete (%d or %d), got %d -- never a torn count", firstBatch, firstBatch+secondBatch, count)
+ require.GreaterOrEqual(t, count, int64(firstBatch), "data durable before the cut must survive the abandonment")
+}
+
+// TestSeriesIndex_CrashCut_MidExternalReceive_ReopenedRootIsComplete pins
+// NIDX-03 §7/§12 item 6: a receive that never reaches CompleteSegment (the
+// window a crash mid external receive would cut through) must never become
+// visible, and -- unlike a graceful Close, which runs abortExternalStreamers
+// and deletes the staged file -- a real crash leaves the staged
+// ".native-external-*" file on disk exactly where WriteChunk left it. This
+// abandons the streamer without calling Close (si is never closed) so the
+// staged file survives, then asserts it is still there, that data admitted
+// and made durable before the cut survives reopen, and that the abandoned
+// segment's data never becomes visible.
+func TestSeriesIndex_CrashCut_MidExternalReceive_ReopenedRootIsComplete(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ // Data durable before the cut: a synchronous (flushTimeoutSeconds == 0)
+ // Insert blocks until this is on disk.
+ survivorIdentity := externalSegIdentity(t, "survives-the-cut")
+ require.NoError(t, si.Insert(index.Documents{{EntityValues: survivorIdentity, Timestamp: 1}}))
+
+ abandonedIdentity := externalSegIdentity(t, "abandoned-mid-receive")
+ seg := buildNativeExternalSegment(t, abandonedIdentity, "value")
+
+ streamer, err := si.EnableExternalSegments()
+ require.NoError(t, err)
+ require.NoError(t, streamer.StartSegment())
+ require.NoError(t, streamer.WriteChunk(seg))
+ // No CompleteSegment, no Close: the receive and the owner are both
+ // abandoned here, exactly as a crash would abandon them.
+
+ indexPath := filepath.Join(dir, seriesIndexDirName)
+ staged, globErr := filepath.Glob(filepath.Join(indexPath, ".native-external-*"))
+ require.NoError(t, globErr)
+ require.Len(t, staged, 1, "the abandoned receive must leave its staged file on disk, exactly as a crash would")
+ _, statErr := os.Stat(staged[0])
+ require.NoError(t, statErr, "the staged file must still exist")
+
+ si2, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si2.Close()) }()
+
+ generation, err := native.OpenReadOnlyGeneration(indexPath)
+ require.NoError(t, err, "a directory abandoned mid-receive must still reopen as a complete generation")
+ defer func() { _ = generation.Close() }()
+ count, err := generation.VisibleDocCount()
+ require.NoError(t, err)
+ require.Equal(t, int64(1), count, "only the data durable before the cut must be visible")
+
+ var survivor, abandoned pbv1.Series
+ require.NoError(t, survivor.Unmarshal(survivorIdentity))
+ require.NoError(t, abandoned.Unmarshal(abandonedIdentity))
+ sd, filterErr := si2.filter(ctx, []*pbv1.Series{&survivor})
+ require.NoError(t, filterErr)
+ require.Len(t, sd.SeriesList, 1, "data durable before the cut must survive")
+
+ sd, filterErr = si2.filter(ctx, []*pbv1.Series{&abandoned})
+ require.NoError(t, filterErr)
+ require.Empty(t, sd.SeriesList, "an incomplete external receive must not become visible after reopen")
+}
diff --git a/banyand/internal/storage/index_external_segments_test.go b/banyand/internal/storage/index_external_segments_test.go
new file mode 100644
index 000000000..c0822caf3
--- /dev/null
+++ b/banyand/internal/storage/index_external_segments_test.go
@@ -0,0 +1,239 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// externalSegIdentity marshals one series identity (a fixed subject
+// testSubjectSvc plus a single string entity value), the same shape
+// buildTestSeriesDocs uses.
+func externalSegIdentity(t *testing.T, entityValue string) []byte {
+ t.Helper()
+ var series pbv1.Series
+ series.Subject = testSubjectSvc
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: entityValue}}}}
+ require.NoError(t, series.Marshal())
+ return append([]byte(nil), series.Buffer...)
+}
+
+// singleSegFile returns the bytes of the one *.seg file under dir.
+func singleSegFile(t *testing.T, dir string) []byte {
+ t.Helper()
+ matches, err := filepath.Glob(filepath.Join(dir, "*.seg"))
+ require.NoError(t, err)
+ require.Len(t, matches, 1, "expected exactly one segment file in %s", dir)
+ data, err := os.ReadFile(matches[0])
+ require.NoError(t, err)
+ return data
+}
+
+// buildNativeExternalSegment creates a fresh native owner at a temp dir,
+// admits one document carrying tag "marker"=markerValue under identity, and
+// returns the resulting committed segment's raw bytes -- a stand-in for a
+// segment the new binary's writer produced.
+func buildNativeExternalSegment(t *testing.T, identity []byte, markerValue string) []byte {
+ t.Helper()
+ dir := t.TempDir()
+ owner, err := native.NewOwner(native.OwnerOptions{Lease: &testRootLease{}, Path: dir, IdentifierDocValues: true})
+ require.NoError(t, err)
+ done := make(chan error, 1)
+ require.NoError(t, owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{
+ Identifier: identity,
+ Fields: []native.Field{{Name: "marker", Value: []byte(markerValue), Store: true}},
+ }},
+ PersistentCallback: func(err error) { done <- err },
+ }))
+ require.NoError(t, <-done)
+ require.NoError(t, owner.Close())
+ return singleSegFile(t, dir)
+}
+
+// buildLegacyExternalSegment creates a fresh *previous-release* series store
+// (pkg/index/inverted) at a temp dir, inserts one document carrying tag
+// "marker"=markerValue under identity, and returns the resulting committed
+// segment's raw bytes -- a stand-in for a segment the previous release's
+// writer produced. Using the legacy writer here is the same sanctioned
+// exception the NIDX-03 fixture generator uses (design §12 item 1): this
+// test's whole point is proving the native external receiver still accepts
+// what the previous release wrote.
+func buildLegacyExternalSegment(t *testing.T, identity []byte, markerValue string) []byte {
+ t.Helper()
+ dir := t.TempDir()
+ store, err := inverted.NewStore(inverted.StoreOpts{Path: dir, BatchWaitSec: 0})
+ require.NoError(t, err)
+ tag := index.NewBytesField(index.FieldKey{TagName: "marker"}, []byte(markerValue))
+ tag.Store = true
+ require.NoError(t, store.UpdateSeriesBatch(index.Batch{Documents: index.Documents{{
+ Fields: []index.Field{tag}, EntityValues: identity,
+ }}}))
+ require.NoError(t, store.Close())
+ return singleSegFile(t, dir)
+}
+
+// receiveExternalSegment drives one StartSegment/WriteChunk/CompleteSegment
+// cycle against idx's external-segment receiver with payload as the whole
+// (single-chunk) segment body.
+func receiveExternalSegment(t *testing.T, idx IndexDB, payload []byte) error {
+ t.Helper()
+ streamer, err := idx.EnableExternalSegments()
+ require.NoError(t, err)
+ require.NoError(t, streamer.StartSegment())
+ if len(payload) > 0 {
+ require.NoError(t, streamer.WriteChunk(payload))
+ }
+ return streamer.CompleteSegment()
+}
+
+// TestSeriesIndex_ExternalReceive_AcceptsLegacyAndNativeSegments pins NIDX-03
+// §7/§12 item 7: raw segments written by the previous release and by native
+// are both accepted by the same external-segment receiver, and both become
+// searchable.
+func TestSeriesIndex_ExternalReceive_AcceptsLegacyAndNativeSegments(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ identityNative := externalSegIdentity(t, "native-written")
+ identityLegacy := externalSegIdentity(t, "legacy-written")
+
+ nativeSeg := buildNativeExternalSegment(t, identityNative, "from-native")
+ legacySeg := buildLegacyExternalSegment(t, identityLegacy, "from-legacy")
+
+ require.NoError(t, receiveExternalSegment(t, si, nativeSeg))
+ require.NoError(t, receiveExternalSegment(t, si, legacySeg))
+
+ var seriesNative, seriesLegacy pbv1.Series
+ require.NoError(t, seriesNative.Unmarshal(identityNative))
+ require.NoError(t, seriesLegacy.Unmarshal(identityLegacy))
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&seriesNative, &seriesLegacy}, IndexSearchOpts{
+ Projection: []index.FieldKey{{TagName: "marker"}},
+ })
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 2)
+ got := map[string]string{}
+ for i, s := range sd.SeriesList {
+ got[s.EntityValues[0].GetStr().GetValue()] = string(sd.Fields[i]["marker"])
+ }
+ require.Equal(t, "from-native", got["native-written"])
+ require.Equal(t, "from-legacy", got["legacy-written"])
+}
+
+// TestSeriesIndex_ExternalReceive_DuplicateIdentifierKeepsExisting pins
+// NIDX-03 §4.3/§7/§12 item 7: a duplicate _id arriving via external receive
+// keeps the already-live document (ExternalDedupKeepExisting), matching the
+// previous release's receiver behavior.
+func TestSeriesIndex_ExternalReceive_DuplicateIdentifierKeepsExisting(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ identity := externalSegIdentity(t, "dup")
+ first := buildNativeExternalSegment(t, identity, "first")
+ second := buildNativeExternalSegment(t, identity, "second")
+
+ require.NoError(t, receiveExternalSegment(t, si, first))
+ require.NoError(t, receiveExternalSegment(t, si, second))
+
+ var series pbv1.Series
+ require.NoError(t, series.Unmarshal(identity))
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&series}, IndexSearchOpts{Projection: []index.FieldKey{{TagName: "marker"}}})
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 1, "a duplicate identifier must not create a second live document")
+ require.Equal(t, "first", string(sd.Fields[0]["marker"]), "the existing document must win over the incoming duplicate")
+}
+
+// TestSeriesIndex_ExternalReceive_LegacyDuplicateKeepsExisting pins the F4
+// combination NIDX-03 §4.3/§7/§12 item 7 implies but the two tests above
+// only establish separately: AcceptsLegacyAndNativeSegments proves a
+// previous-release segment is accepted and searchable, and
+// DuplicateIdentifierKeepsExisting proves the dedup rule using two native
+// segments. Neither, on its own, proves the dedup rule also holds when the
+// already-live document came from the legacy writer. This test receives a
+// legacy-written segment first and a native-written duplicate of the same
+// identifier second, through the same StartSegment/WriteChunk/
+// CompleteSegment receiver both other tests use: the receiver's dedup logic
+// (ExternalDedupKeepExisting) operates on parsed documents and never
+// branches on which writer produced the bytes, so the legacy-authored
+// document must still win.
+func TestSeriesIndex_ExternalReceive_LegacyDuplicateKeepsExisting(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ identity := externalSegIdentity(t, "legacy-dup")
+ first := buildLegacyExternalSegment(t, identity, "legacy-first")
+ second := buildNativeExternalSegment(t, identity, "native-second")
+
+ require.NoError(t, receiveExternalSegment(t, si, first))
+ require.NoError(t, receiveExternalSegment(t, si, second))
+
+ var series pbv1.Series
+ require.NoError(t, series.Unmarshal(identity))
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&series}, IndexSearchOpts{Projection: []index.FieldKey{{TagName: "marker"}}})
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 1, "a duplicate identifier must not create a second live document")
+ require.Equal(t, "legacy-first", string(sd.Fields[0]["marker"]), "a legacy-written existing document must also win over an incoming native duplicate")
+}
+
+// TestSeriesIndex_ExternalReceive_TruncatedSegmentRejectedNothingVisible pins
+// NIDX-03 §7/§12 item 7: a truncated (corrupt) segment fails validation
+// before introduction and leaves nothing visible.
+func TestSeriesIndex_ExternalReceive_TruncatedSegmentRejectedNothingVisible(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ identity := externalSegIdentity(t, "truncated")
+ seg := buildNativeExternalSegment(t, identity, "value")
+ truncated := seg[:len(seg)/2]
+
+ receiveErr := receiveExternalSegment(t, si, truncated)
+ require.Error(t, receiveErr, "a truncated segment must be rejected")
+
+ var series pbv1.Series
+ require.NoError(t, series.Unmarshal(identity))
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&series}, IndexSearchOpts{})
+ require.NoError(t, err)
+ require.Empty(t, sd.SeriesList, "a rejected segment must leave nothing visible")
+}
diff --git a/banyand/internal/storage/index_fixture_generate_test.go b/banyand/internal/storage/index_fixture_generate_test.go
new file mode 100644
index 000000000..0db22ed84
--- /dev/null
+++ b/banyand/internal/storage/index_fixture_generate_test.go
@@ -0,0 +1,206 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// nidx03FixtureDir is the checked-in sidx fixture directory pair
+// (normal/sidx, indexmode/sidx) TestSeriesIndex_OpensPreviousReleaseFixture
+// opens with the native series index. See
+// nidx03FixtureNormalDocs/nidx03FixtureIndexModeDocs below for the exact,
+// hand-authored document set these directories hold, and
+// index_fixture_verify_test.go for the pinned expected query results.
+const nidx03FixtureDir = "testdata/nidx03_fixture"
+
+// nidx03ScoreRuleID is the index-rule ID the fixture's "score" field is
+// keyed by, so the sort test can exercise the real OrderByTypeIndex path
+// (FieldKey{IndexRuleID}.Marshal()) the same way a production Measure
+// index-rule sort does, instead of a TagName-keyed field no OrderBy path
+// addresses.
+const nidx03ScoreRuleID = 42
+
+// nidx03FixtureSeries describes one fixture series: a two-tag entity
+// (service, instance) plus its tag field values. A nil map value for a
+// field name means "this series has no such field" (exercises NOT_HAVING /
+// absent-field projection).
+//
+//nolint:govet // fixture fields are grouped by meaning, not padding.
+type nidx03FixtureSeries struct {
+ tags []string
+ service string
+ instance string
+ region string
+ score string // zero-padded decimal so byte order == numeric order.
+ description string
+ timestamp int64
+ version int64
+}
+
+// nidx03FixtureNormalDocs is the hand-authored fixture content for the
+// non-index-mode sidx: four series covering exact/prefix/wildcard identity
+// matching (shared "alpha"/"one" components), every criteria.Filter kind
+// (EQ/range/IN/NOT_IN/HAVING/NOT_HAVING/MATCH) and a sortable field. Kept
+// deliberately tiny (4 series) so the checked-in fixture stays a few KB.
+func nidx03FixtureNormalDocs() []nidx03FixtureSeries {
+ return []nidx03FixtureSeries{
+ {service: "alpha", instance: "one", region: "us", score: "010", tags: []string{"a", "b"}, description: "quick brown fox", timestamp: 1000, version: 1},
+ {service: "alpha", instance: "two", region: "us", score: "020", tags: []string{"b", "c"}, description: "lazy dog sleeps", timestamp: 2000, version: 1},
+ {service: "beta", instance: "one", region: "eu", score: "030", tags: []string{"a"}, description: "quick fox jumps", timestamp: 3000, version: 2},
+ {service: "gamma", instance: "one", region: "us", score: "005", tags: nil, description: "dog barks loud", timestamp: 4000, version: 1},
+ }
+}
+
+// nidx03FixtureIndexModeDocs is the hand-authored fixture content for the
+// index-mode sidx: two series under subject "idx_measure", one under a
+// different subject "other_measure" to prove SearchWithoutSeries' _im_name
+// universe scoping excludes it.
+func nidx03FixtureIndexModeDocs() []nidx03FixtureSeries {
+ return []nidx03FixtureSeries{
+ {service: "svcA", instance: "", region: "us", score: "001", timestamp: 1000, version: 1},
+ {service: "svcB", instance: "", region: "eu", score: "002", timestamp: 2000, version: 1},
+ }
+}
+
+// nidx03FixtureOtherSubjectDoc is the one "other_measure" document the
+// index-mode fixture also carries.
+func nidx03FixtureOtherSubjectDoc() nidx03FixtureSeries {
+ return nidx03FixtureSeries{service: "svcC", instance: "", region: "us", score: "003", timestamp: 3000, version: 1}
+}
+
+// buildNormalFixtureDoc builds the legacy-store index.Document for one
+// normal-mode fixture series, matching storage.EncodeSeriesDocument's
+// mapping one-for-one so the native reader interprets it identically.
+func buildNormalFixtureDoc(t *testing.T, s nidx03FixtureSeries) index.Document {
+ t.Helper()
+ var series pbv1.Series
+ series.Subject = testSubjectSvc
+ series.EntityValues = []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: s.service}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: s.instance}}},
+ }
+ require.NoError(t, series.Marshal())
+
+ var fields []index.Field
+ region := index.NewBytesField(index.FieldKey{TagName: "region"}, []byte(s.region))
+ region.Store, region.Index, region.NoSort = true, true, true
+ fields = append(fields, region)
+
+ score := index.NewBytesField(index.FieldKey{IndexRuleID: nidx03ScoreRuleID}, []byte(s.score))
+ score.Store, score.Index = true, true // Sort (NoSort=false): score is the fixture's sort-order field.
+ fields = append(fields, score)
+
+ for _, tag := range s.tags {
+ tf := index.NewBytesField(index.FieldKey{TagName: "tag"}, []byte(tag))
+ tf.Store, tf.Index, tf.NoSort = true, true, true
+ fields = append(fields, tf)
+ }
+
+ desc := index.NewStringField(index.FieldKey{TagName: "description", Analyzer: index.AnalyzerSimple}, s.description)
+ desc.Store, desc.Index, desc.NoSort = true, true, true
+ fields = append(fields, desc)
+
+ return index.Document{
+ Fields: fields,
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Timestamp: s.timestamp,
+ Version: s.version,
+ }
+}
+
+// buildIndexModeFixtureDoc builds the legacy-store index.Document for one
+// index-mode fixture series: an _im_name field (the subject, matching
+// appendEntityTagsToIndexFields' production shape) plus a regular "region"/
+// "score" tag, keyed the same way buildNormalFixtureDoc does.
+func buildIndexModeFixtureDoc(t *testing.T, subject string, s nidx03FixtureSeries) index.Document {
+ t.Helper()
+ var series pbv1.Series
+ series.Subject = subject
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: s.service}}}}
+ require.NoError(t, series.Marshal())
+
+ imName := index.NewStringField(index.FieldKey{TagName: index.IndexModeName}, subject)
+ imName.Index, imName.NoSort = true, true
+
+ region := index.NewBytesField(index.FieldKey{TagName: "region"}, []byte(s.region))
+ region.Store, region.Index, region.NoSort = true, true, true
+
+ score := index.NewBytesField(index.FieldKey{IndexRuleID: nidx03ScoreRuleID}, []byte(s.score))
+ score.Store, score.Index = true, true
+
+ return index.Document{
+ Fields: []index.Field{imName, region, score},
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Timestamp: s.timestamp,
+ Version: s.version,
+ }
+}
+
+// TestGenerateNIDX03Fixture is the NIDX-03 §12 item 1 one-off generator. It
+// is skipped unless GENERATE_NIDX03_FIXTURE is set, so it never runs in CI;
+// invoke it once (GENERATE_NIDX03_FIXTURE=1 go test -run
+// TestGenerateNIDX03Fixture ./banyand/internal/storage/ -v) to (re)produce
+// the checked-in fixture under testdata/nidx03_fixture using the CURRENT
+// legacy store (pkg/index/inverted), exactly as the previous release's
+// writer would. It is not kept running in CI; delete it once the fixture is
+// pinned and stable, or leave it here, permanently skipped, as the record
+// of how the fixture was produced.
+func TestGenerateNIDX03Fixture(t *testing.T) {
+ if os.Getenv("GENERATE_NIDX03_FIXTURE") == "" {
+ t.Skip("one-off fixture generator; set GENERATE_NIDX03_FIXTURE=1 to (re)run it")
+ }
+
+ root, err := filepath.Abs(nidx03FixtureDir)
+ require.NoError(t, err)
+ require.NoError(t, os.RemoveAll(root))
+
+ normalDir := filepath.Join(root, "normal", "sidx")
+ require.NoError(t, os.MkdirAll(normalDir, 0o755))
+ normalStore, err := inverted.NewStore(inverted.StoreOpts{Path: normalDir, BatchWaitSec: 0})
+ require.NoError(t, err)
+ var normalDocs index.Documents
+ for _, s := range nidx03FixtureNormalDocs() {
+ normalDocs = append(normalDocs, buildNormalFixtureDoc(t, s))
+ }
+ require.NoError(t, normalStore.UpdateSeriesBatch(index.Batch{Documents: normalDocs}))
+ require.NoError(t, normalStore.Close())
+
+ indexModeDir := filepath.Join(root, "indexmode", "sidx")
+ require.NoError(t, os.MkdirAll(indexModeDir, 0o755))
+ indexModeStore, err := inverted.NewStore(inverted.StoreOpts{Path: indexModeDir, BatchWaitSec: 0})
+ require.NoError(t, err)
+ var indexModeDocs index.Documents
+ for _, s := range nidx03FixtureIndexModeDocs() {
+ indexModeDocs = append(indexModeDocs, buildIndexModeFixtureDoc(t, "idx_measure", s))
+ }
+ indexModeDocs = append(indexModeDocs, buildIndexModeFixtureDoc(t, "other_measure", nidx03FixtureOtherSubjectDoc()))
+ require.NoError(t, indexModeStore.UpdateSeriesBatch(index.Batch{Documents: indexModeDocs}))
+ require.NoError(t, indexModeStore.Close())
+
+ t.Logf("fixture written to %s", root)
+}
diff --git a/banyand/internal/storage/index_fixture_verify_test.go b/banyand/internal/storage/index_fixture_verify_test.go
new file mode 100644
index 000000000..6659e3d58
--- /dev/null
+++ b/banyand/internal/storage/index_fixture_verify_test.go
@@ -0,0 +1,310 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// fixtureResolver implements criteria.FieldResolver for the NIDX-03 fixture:
+// "region"/"tag" are TagName-keyed fields (their own name is the engine
+// field); "score" is IndexRuleID-keyed (nidx03ScoreRuleID), the same way a
+// production Measure index-rule field is, so the sort test below can
+// exercise the real OrderByTypeIndex path; "description" carries the simple
+// analyzer MATCH needs.
+type fixtureResolver struct{}
+
+func (fixtureResolver) Field(tagName string) (string, string, bool) {
+ switch tagName {
+ case "region", "tag":
+ return tagName, "", true
+ case "score":
+ return index.FieldKey{IndexRuleID: nidx03ScoreRuleID}.Marshal(), "", true
+ case "description":
+ return tagName, index.AnalyzerSimple, true
+ default:
+ return "", "", false
+ }
+}
+
+func fixtureStrTagValue(v string) *modelv1.TagValue {
+ return &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: v}}}
+}
+
+func fixtureSeriesExact(service, instance string) *pbv1.Series {
+ return &pbv1.Series{Subject: testSubjectSvc, EntityValues: []*modelv1.TagValue{fixtureStrTagValue(service), fixtureStrTagValue(instance)}}
+}
+
+func fixtureSeriesPrefix(service string) *pbv1.Series {
+ return &pbv1.Series{Subject: testSubjectSvc, EntityValues: []*modelv1.TagValue{fixtureStrTagValue(service), pbv1.AnyTagValue}}
+}
+
+func fixtureSeriesWildcard(instance string) *pbv1.Series {
+ return &pbv1.Series{Subject: testSubjectSvc, EntityValues: []*modelv1.TagValue{pbv1.AnyTagValue, fixtureStrTagValue(instance)}}
+}
+
+func fixtureCondition(name string, op modelv1.Condition_BinaryOp, value *modelv1.TagValue) *modelv1.Criteria {
+ return &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{Name: name, Op: op, Value: value}}}
+}
+
+func fixtureStrArray(values ...string) *modelv1.TagValue {
+ return &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: values}}}
+}
+
+// copyDirForTest recursively byte-copies src into dst (both must be plain
+// directories of regular files), so a test can open a writable owner on a
+// disposable copy of a checked-in fixture instead of risking any mutation of
+// the fixture itself.
+func copyDirForTest(t *testing.T, src, dst string) {
+ t.Helper()
+ entries, err := os.ReadDir(src)
+ require.NoError(t, err)
+ require.NoError(t, os.MkdirAll(dst, 0o755))
+ for _, entry := range entries {
+ srcPath := filepath.Join(src, entry.Name())
+ dstPath := filepath.Join(dst, entry.Name())
+ if entry.IsDir() {
+ copyDirForTest(t, srcPath, dstPath)
+ continue
+ }
+ data, readErr := os.ReadFile(srcPath)
+ require.NoError(t, readErr)
+ require.NoError(t, os.WriteFile(dstPath, data, 0o600))
+ }
+}
+
+// openNIDX03Fixture copies the checked-in fixture at testdata/nidx03_fixture/
+// <name> into a fresh temp dir and opens it with the native series index
+// (newSeriesIndex), exactly as the new binary opens a sidx directory the
+// previous release wrote (NIDX-03 §1's "Forward" compatibility guarantee).
+func openNIDX03Fixture(t *testing.T, name string) *seriesIndex {
+ t.Helper()
+ fixtureRoot, err := filepath.Abs(nidx03FixtureDir)
+ require.NoError(t, err)
+ src := filepath.Join(fixtureRoot, name)
+ require.DirExists(t, src, "fixture missing; run TestGenerateNIDX03Fixture with GENERATE_NIDX03_FIXTURE=1")
+
+ workDir := t.TempDir()
+ copyDirForTest(t, src, workDir)
+
+ si, err := newSeriesIndex(context.Background(), workDir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, si.Close()) })
+ return si
+}
+
+// searchSeries runs si.Search(series, criteria) with the fixture resolver and
+// returns the matched (service, instance) pairs sorted for stable
+// comparison.
+func searchFixtureSeries(t *testing.T, si *seriesIndex, series []*pbv1.Series, criteria *modelv1.Criteria) []string {
+ t.Helper()
+ sd, _, err := si.Search(context.Background(), series, IndexSearchOpts{Criteria: criteria, Fields: fixtureResolver{}})
+ require.NoError(t, err)
+ return sortedFixtureLabels(sd.SeriesList)
+}
+
+// fixtureLabels renders sl as "<service>/<instance>" labels, preserving sl's
+// own order (which, for a sorted Search, is the result's actual sort order
+// and must not be disturbed).
+func fixtureLabels(sl pbv1.SeriesList) []string {
+ labels := make([]string, 0, len(sl))
+ for _, s := range sl {
+ service := s.EntityValues[0].GetStr().GetValue()
+ instance := ""
+ if len(s.EntityValues) > 1 {
+ instance = s.EntityValues[1].GetStr().GetValue()
+ }
+ labels = append(labels, service+"/"+instance)
+ }
+ return labels
+}
+
+// sortedFixtureLabels is fixtureLabels with the labels themselves sorted, for
+// comparing an unordered filter result (Search/SearchWithoutSeries with no
+// Order) where only set membership, not result order, is pinned.
+func sortedFixtureLabels(sl pbv1.SeriesList) []string {
+ labels := fixtureLabels(sl)
+ sort.Strings(labels)
+ return labels
+}
+
+// TestSeriesIndex_OpensPreviousReleaseFixture_NormalMode is NIDX-03 §12 item
+// 1: the fixture testdata/nidx03_fixture/normal/sidx, produced by the
+// CURRENT legacy store (TestGenerateNIDX03Fixture), is opened by the native
+// series index and every canned query -- exact, prefix, wildcard, and every
+// criteria.Filter kind -- returns exactly the pinned expected result,
+// proving the previous release's on-disk sidx is read identically.
+//
+// Expected results below (per design §12.1, "literal expectations" from a
+// legacy-store run) were cross-checked against an independent, direct read
+// of the checked-in fixture's raw stored fields with the legacy bluge
+// reader (bluge.OpenReader + DocumentMatch.VisitStoredFields, computing
+// each scenario's matches programmatically from the on-disk data rather
+// than from nidx03FixtureNormalDocs' declared Go literal) before being
+// pinned here; every one matched exactly.
+func TestSeriesIndex_OpensPreviousReleaseFixture_NormalMode(t *testing.T) {
+ si := openNIDX03Fixture(t, "normal")
+ all := []*pbv1.Series{
+ fixtureSeriesExact("alpha", "one"), fixtureSeriesExact("alpha", "two"),
+ fixtureSeriesExact("beta", "one"), fixtureSeriesExact("gamma", "one"),
+ }
+
+ t.Run("exact", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, []*pbv1.Series{fixtureSeriesExact("alpha", "one")}, nil)
+ require.Equal(t, []string{"alpha/one"}, got)
+ })
+ t.Run("prefix", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, []*pbv1.Series{fixtureSeriesPrefix("alpha")}, nil)
+ require.Equal(t, []string{"alpha/one", "alpha/two"}, got)
+ })
+ t.Run("wildcard", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, []*pbv1.Series{fixtureSeriesWildcard("one")}, nil)
+ require.Equal(t, []string{"alpha/one", "beta/one", "gamma/one"}, got)
+ })
+ t.Run("EQ", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("region", modelv1.Condition_BINARY_OP_EQ, fixtureStrTagValue("us")))
+ require.Equal(t, []string{"alpha/one", "alpha/two", "gamma/one"}, got)
+ })
+ t.Run("range GT", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("score", modelv1.Condition_BINARY_OP_GT, fixtureStrTagValue("010")))
+ require.Equal(t, []string{"alpha/two", "beta/one"}, got)
+ })
+ t.Run("range LE", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("score", modelv1.Condition_BINARY_OP_LE, fixtureStrTagValue("010")))
+ require.Equal(t, []string{"alpha/one", "gamma/one"}, got)
+ })
+ t.Run("IN", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("region", modelv1.Condition_BINARY_OP_IN, fixtureStrArray("eu", "xx")))
+ require.Equal(t, []string{"beta/one"}, got)
+ })
+ t.Run("NOT_IN", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("region", modelv1.Condition_BINARY_OP_NOT_IN, fixtureStrArray("eu")))
+ require.Equal(t, []string{"alpha/one", "alpha/two", "gamma/one"}, got)
+ })
+ t.Run("HAVING", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("tag", modelv1.Condition_BINARY_OP_HAVING, fixtureStrArray("a", "b")))
+ require.Equal(t, []string{"alpha/one"}, got, "only alpha/one carries both tag=a and tag=b")
+ })
+ t.Run("NOT_HAVING", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("tag", modelv1.Condition_BINARY_OP_NOT_HAVING, fixtureStrArray("a")))
+ require.Equal(t, []string{"alpha/two", "gamma/one"}, got, "alpha/one and beta/one carry tag=a; gamma/one carries no tag field at all")
+ })
+ t.Run("MATCH", func(t *testing.T) {
+ got := searchFixtureSeries(t, si, all, fixtureCondition("description", modelv1.Condition_BINARY_OP_MATCH, fixtureStrTagValue("fox")))
+ require.Equal(t, []string{"alpha/one", "beta/one"}, got)
+ })
+ t.Run("sort by score ascending", func(t *testing.T) {
+ sd, sortedValues, err := si.Search(context.Background(), all, IndexSearchOpts{
+ Order: &index.OrderBy{
+ Type: index.OrderByTypeIndex,
+ Index: &databasev1.IndexRule{Metadata: &commonv1.Metadata{Id: nidx03ScoreRuleID}},
+ Sort: modelv1.Sort_SORT_ASC,
+ },
+ })
+ require.NoError(t, err)
+ require.Equal(t, []string{"gamma/one", "alpha/one", "alpha/two", "beta/one"}, fixtureLabels(sd.SeriesList))
+ require.Len(t, sortedValues, 4)
+ for i := 1; i < len(sortedValues); i++ {
+ require.LessOrEqual(t, string(sortedValues[i-1]), string(sortedValues[i]), "sorted values must be non-decreasing")
+ }
+ })
+ t.Run("timestamps and versions", func(t *testing.T) {
+ sd, _, err := si.Search(context.Background(), all, IndexSearchOpts{})
+ require.NoError(t, err)
+ gotTimestamps := make(map[string]int64, len(sd.SeriesList))
+ gotVersions := make(map[string]int64, len(sd.SeriesList))
+ for i, label := range fixtureLabels(sd.SeriesList) {
+ require.True(t, sd.TimestampSet[i], "fixture series %s must carry a timestamp", label)
+ require.True(t, sd.VersionSet[i], "fixture series %s must carry a version", label)
+ gotTimestamps[label] = sd.Timestamps[i]
+ gotVersions[label] = sd.Versions[i]
+ }
+ // From nidx03FixtureNormalDocs' declared content.
+ require.Equal(t, map[string]int64{"alpha/one": 1000, "alpha/two": 2000, "beta/one": 3000, "gamma/one": 4000}, gotTimestamps)
+ require.Equal(t, map[string]int64{"alpha/one": 1, "alpha/two": 1, "beta/one": 2, "gamma/one": 1}, gotVersions)
+ })
+}
+
+// TestSeriesIndex_OpensPreviousReleaseFixture_IndexMode is NIDX-03 §12 item 1
+// (index-mode Measure): the fixture testdata/nidx03_fixture/indexmode/sidx
+// holds two "idx_measure" series and one "other_measure" series;
+// SearchWithoutSeries scoped to IndexModeSubject "idx_measure" must return
+// exactly the two, never the third (proving the _im_name universe scoping).
+func TestSeriesIndex_OpensPreviousReleaseFixture_IndexMode(t *testing.T) {
+ si := openNIDX03Fixture(t, "indexmode")
+
+ sd, _, err := si.SearchWithoutSeries(context.Background(), IndexSearchOpts{
+ IndexModeSubject: "idx_measure",
+ Projection: []index.FieldKey{{TagName: "region"}},
+ })
+ require.NoError(t, err)
+ labels := make(map[string]string, len(sd.SeriesList))
+ for i, s := range sd.SeriesList {
+ labels[s.EntityValues[0].GetStr().GetValue()] = string(sd.Fields[i][index.FieldKey{TagName: "region"}.Marshal()])
+ }
+ require.Equal(t, map[string]string{"svcA": "us", "svcB": "eu"}, labels, "only idx_measure's own series must be returned, with their region values")
+
+ // Index-mode criteria: a region filter within the idx_measure universe
+ // must apply on top of (not instead of) the _im_name scoping -- svcC
+ // (region=us) belongs to "other_measure" and must stay excluded even
+ // though its region matches.
+ t.Run("criteria", func(t *testing.T) {
+ sd, _, err := si.SearchWithoutSeries(context.Background(), IndexSearchOpts{
+ IndexModeSubject: "idx_measure",
+ Criteria: fixtureCondition("region", modelv1.Condition_BINARY_OP_EQ, fixtureStrTagValue("us")),
+ Fields: fixtureResolver{},
+ })
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 1)
+ require.Equal(t, "svcA", sd.SeriesList[0].EntityValues[0].GetStr().GetValue())
+ })
+
+ // Index-mode sort: svcA (score=001) must sort before svcB (score=002)
+ // within the idx_measure universe; svcC (other_measure) must stay
+ // excluded from both the result set and the sort.
+ t.Run("sort", func(t *testing.T) {
+ sd, sortedValues, err := si.SearchWithoutSeries(context.Background(), IndexSearchOpts{
+ IndexModeSubject: "idx_measure",
+ Order: &index.OrderBy{
+ Type: index.OrderByTypeIndex,
+ Index: &databasev1.IndexRule{Metadata: &commonv1.Metadata{Id: nidx03ScoreRuleID}},
+ Sort: modelv1.Sort_SORT_ASC,
+ },
+ })
+ require.NoError(t, err)
+ got := make([]string, len(sd.SeriesList))
+ for i, s := range sd.SeriesList {
+ got[i] = s.EntityValues[0].GetStr().GetValue()
+ }
+ require.Equal(t, []string{"svcA", "svcB"}, got)
+ require.Len(t, sortedValues, 2)
+ require.LessOrEqual(t, string(sortedValues[0]), string(sortedValues[1]), "sorted values must be non-decreasing")
+ })
+}
diff --git a/banyand/internal/storage/index_reference_model_test.go b/banyand/internal/storage/index_reference_model_test.go
new file mode 100644
index 000000000..471de4966
--- /dev/null
+++ b/banyand/internal/storage/index_reference_model_test.go
@@ -0,0 +1,526 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "os"
+ "sort"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// refModelScoreRuleID is the fixed IndexRuleID the reference model's "score"
+// field carries, so the sorted-search check below exercises the real
+// OrderByTypeIndex path (the same one dispatch.go's resolveOrderBy
+// produces), not the time order.
+const refModelScoreRuleID = uint32(901)
+
+// refModelDoc is one reference-model document: the field values a real
+// index.Document.Fields list would carry (keyed by marshaled FieldKey name),
+// the set of field names it carries (what InsertIfAbsent's coverage check
+// compares), its timestamp/version, and its zero-padded "score" value (for
+// the index-rule sorted-search check; zero-padding makes lexicographic byte
+// order equal numeric order, so SortedValue's raw bytes can be compared
+// directly).
+type refModelDoc struct {
+ fields map[string][]byte
+ fieldSet map[string]struct{}
+ score string
+ timestamp int64
+ version int64
+}
+
+// refModel is the plain in-memory model of NIDX-03 §2: a map from _id to
+// document, plus the external-receive keep-existing rule (§4.3). It never
+// touches the series index itself; TestSeriesIndex_ReferenceModel drives
+// both in lockstep and compares.
+//
+// Insert (§2.1's segment-wide field-set coverage skip) is deliberately NOT
+// exercised by this randomized model: pkg/index/native.Owner computes
+// InsertIfAbsent's coverage check per SEGMENT (the union of every document's
+// field names the segment physically holds), not per identifier, and
+// newSeriesIndex's production config runs compaction (CompactionThreshold
+// left at its conservative default), which repeatedly merges many series'
+// segments into one over the run -- a simple per-identifier model cannot
+// predict the resulting segment-wide field set without reimplementing the
+// engine's own merge decisions. TestSeriesIndex_Insert_FieldCoverageContract
+// below pins Insert's contract directly instead, staying well under the
+// compaction threshold so segment-wide and per-identifier coverage coincide.
+type refModel struct {
+ docs map[string]*refModelDoc
+}
+
+func newRefModel() *refModel { return &refModel{docs: map[string]*refModelDoc{}} }
+
+// applyUpdate mirrors Update's contract: full replace, no version check.
+func (m *refModel) applyUpdate(identity string, doc *refModelDoc) {
+ m.docs[identity] = doc
+}
+
+// applyExternalReceive mirrors ExternalDedupKeepExisting: the existing live
+// document wins; an incoming duplicate is masked.
+func (m *refModel) applyExternalReceive(identity string, doc *refModelDoc) {
+ if _, ok := m.docs[identity]; ok {
+ return
+ }
+ m.docs[identity] = doc
+}
+
+// refModelFieldNames enumerates the small fixed field catalog the property
+// test draws random subsets from.
+var refModelFieldNames = []string{"region", "zone", "team", "tier"}
+
+// referenceModelIterations lets REFMODEL_ITERATIONS (unset -> 200) drive a
+// longer run out-of-band without touching the default CI budget, matching
+// NIDX-03 §12 item 3's "fixed seed in CI, longer behind a flag" requirement.
+func referenceModelIterations() int {
+ if raw := os.Getenv("REFMODEL_ITERATIONS"); raw != "" {
+ if n, err := strconv.Atoi(raw); err == nil && n > 0 {
+ return n
+ }
+ }
+ return 200
+}
+
+// buildRefModelDoc builds both the index.Document (for the real series
+// index) and the refModelDoc (for the model) for one series, drawing a
+// random subset of refModelFieldNames with random values from rng, plus a
+// zero-padded "score" field every document always carries.
+func buildRefModelDoc(rng *rand.Rand, identity []byte, timestamp, version int64) (index.Document, *refModelDoc) {
+ names := append([]string(nil), refModelFieldNames...)
+ rng.Shuffle(len(names), func(i, j int) { names[i], names[j] = names[j], names[i] })
+ chosen := names[:rng.Intn(len(names)+1)]
+ sort.Strings(chosen) // deterministic catalog order regardless of shuffle draw.
+
+ fields := make([]index.Field, 0, len(chosen)+1)
+ modelFields := make(map[string][]byte, len(chosen)+1)
+ fieldSet := make(map[string]struct{}, len(chosen)+1)
+ for _, name := range chosen {
+ value := []byte(fmt.Sprintf("v%d", rng.Intn(5)))
+ key := index.FieldKey{TagName: name}
+ f := index.NewBytesField(key, value)
+ f.Store = true
+ f.Index = true
+ fields = append(fields, f)
+ modelFields[key.Marshal()] = value
+ fieldSet[key.Marshal()] = struct{}{}
+ }
+
+ score := fmt.Sprintf("%03d", rng.Intn(1000))
+ scoreKey := index.FieldKey{IndexRuleID: refModelScoreRuleID}
+ scoreField := index.NewStringField(scoreKey, score)
+ scoreField.Store = true
+ scoreField.Index = true
+ fields = append(fields, scoreField)
+ modelFields[scoreKey.Marshal()] = []byte(score)
+ fieldSet[scoreKey.Marshal()] = struct{}{}
+
+ doc := index.Document{Fields: fields, EntityValues: identity, Timestamp: timestamp, Version: version}
+ return doc, &refModelDoc{fields: modelFields, fieldSet: fieldSet, score: score, timestamp: timestamp, version: version}
+}
+
+// refModelResolver implements criteria.FieldResolver for the catalog above,
+// so the criteria-filter check can run through the real
+// pkg/index/native/criteria.Filter path.
+type refModelResolver struct{}
+
+func (refModelResolver) Field(tagName string) (string, string, bool) {
+ if tagName == "score" {
+ return index.FieldKey{IndexRuleID: refModelScoreRuleID}.Marshal(), "", true
+ }
+ for _, name := range refModelFieldNames {
+ if name == tagName {
+ return index.FieldKey{TagName: name}.Marshal(), "", true
+ }
+ }
+ return "", "", false
+}
+
+// refModelPopulation is 4 groups x 3 instances: a two-entity-tag-position
+// identity scheme (group, instance) so prefix (fixed group, wildcard
+// instance) and wildcard (wildcard group, fixed instance) series matchers
+// are meaningful, unlike a single-tag-position identity.
+const (
+ refModelGroups = 4
+ refModelInstances = 3
+ refModelPopulation = refModelGroups * refModelInstances
+)
+
+// refModelSeriesIdentity returns the marshaled identity for series index i in
+// the (group, instance) population, and the *pbv1.Series to search for it.
+func refModelSeriesIdentity(t *testing.T, i int) ([]byte, *pbv1.Series) {
+ t.Helper()
+ group, instance := i/refModelInstances, i%refModelInstances
+ var series pbv1.Series
+ series.Subject = "refmodel"
+ series.EntityValues = []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("group-%02d", group)}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("instance-%02d", instance)}}},
+ }
+ require.NoError(t, series.Marshal())
+ return append([]byte(nil), series.Buffer...), &series
+}
+
+// buildNativeExternalSegmentMultiField is buildNativeExternalSegment
+// generalized to carry several already-marshaled fields (keyed by FieldKey
+// name) plus a version, matching what the reference-model property test
+// needs to simulate an external receive of a multi-field, versioned
+// document -- the same "_version" stored-only field
+// storage.EncodeSeriesDocument writes.
+func buildNativeExternalSegmentMultiField(t *testing.T, identity []byte, fields map[string][]byte, timestamp, version int64) []byte {
+ t.Helper()
+ dir := t.TempDir()
+ owner, err := native.NewOwner(native.OwnerOptions{Lease: &testRootLease{}, Path: dir, IdentifierDocValues: true})
+ require.NoError(t, err)
+ nativeFields := make([]native.Field, 0, len(fields)+1)
+ for name, value := range fields {
+ // Sort: true matches storage.EncodeSeriesDocument's mapping for an
+ // indexed field with no NoSort override -- the real sender-side
+ // encoding every product uses, including for data a replication
+ // receiver introduces via external segments. Omitting it here would
+ // leave this fixture's fields un-sortable, unlike what production
+ // ever actually writes.
+ nativeFields = append(nativeFields, native.Field{Name: name, Value: value, Store: true, Index: true, Sort: true})
+ }
+ nativeFields = append(nativeFields, native.Field{Name: versionFieldName, Value: convert.Int64ToBytes(version), Store: true})
+ done := make(chan error, 1)
+ require.NoError(t, owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{Identifier: identity, Fields: nativeFields, Timestamp: timestamp}},
+ PersistentCallback: func(err error) { done <- err },
+ }))
+ require.NoError(t, <-done)
+ require.NoError(t, owner.Close())
+ return singleSegFile(t, dir)
+}
+
+// receiveExternalSegmentRetryingBusy retries receiveExternalSegment on
+// native.ErrPersistenceBusy: introduceExternalSegment legitimately refuses a
+// receive while the owner's own background garbage collection is active
+// (owner.go's o.collecting guard) -- a real, expected contention a
+// production replication receiver hits too, not a test bug -- and
+// CompleteSegment discards the streamer's staged bytes on any error, so the
+// whole StartSegment/WriteChunk/CompleteSegment cycle must be redone with
+// payload, not merely re-invoked.
+func receiveExternalSegmentRetryingBusy(t *testing.T, idx IndexDB, payload []byte) {
+ t.Helper()
+ require.Eventually(t, func() bool {
+ err := receiveExternalSegment(t, idx, payload)
+ if err == nil {
+ return true
+ }
+ require.ErrorIs(t, err, native.ErrPersistenceBusy, "only transient GC contention may be retried")
+ return false
+ }, 5*time.Second, time.Millisecond)
+}
+
+// verifyRefModel re-reads si (unsorted, then index-rule sorted) and asserts
+// its observations against model: live document count and per-document
+// fields/timestamp/version (unsorted), plus -- the index-rule sorted search
+// -- that the SAME live set comes back and SortedValue is non-decreasing
+// and consistent with each row's own "score" field.
+func verifyRefModel(ctx context.Context, t *testing.T, si *seriesIndex, seriesByIdentity []*pbv1.Series, model *refModel) {
+ t.Helper()
+ projection := make([]index.FieldKey, 0, len(refModelFieldNames)+1)
+ for _, name := range refModelFieldNames {
+ projection = append(projection, index.FieldKey{TagName: name})
+ }
+ projection = append(projection, index.FieldKey{IndexRuleID: refModelScoreRuleID})
+
+ sd, _, searchErr := si.Search(ctx, seriesByIdentity, IndexSearchOpts{Projection: projection})
+ require.NoError(t, searchErr)
+
+ got := make(map[string]*refModelDoc, len(sd.SeriesList))
+ for i, s := range sd.SeriesList {
+ s.Buffer = nil // Unmarshal leaves Buffer as decode scratch; Marshal appends rather than resetting.
+ require.NoError(t, s.Marshal())
+ fields := map[string][]byte{}
+ for _, name := range refModelFieldNames {
+ key := index.FieldKey{TagName: name}.Marshal()
+ if v, ok := sd.Fields[i][key]; ok {
+ fields[key] = v
+ }
+ }
+ scoreKey := index.FieldKey{IndexRuleID: refModelScoreRuleID}.Marshal()
+ var score string
+ if v, ok := sd.Fields[i][scoreKey]; ok {
+ fields[scoreKey] = v
+ score = string(v)
+ }
+ var version int64
+ if sd.VersionSet[i] {
+ version = sd.Versions[i]
+ }
+ var timestamp int64
+ if sd.TimestampSet[i] {
+ timestamp = sd.Timestamps[i]
+ }
+ got[string(s.Buffer)] = &refModelDoc{fields: fields, score: score, timestamp: timestamp, version: version}
+ }
+
+ require.Len(t, got, len(model.docs), "live document count must match the model")
+ for identity, want := range model.docs {
+ actual, ok := got[identity]
+ require.True(t, ok, "model expects series %q to be live", identity)
+ require.Equal(t, want.timestamp, actual.timestamp, "series %q timestamp", identity)
+ require.Equal(t, want.version, actual.version, "series %q version", identity)
+ require.Equal(t, want.score, actual.score, "series %q score", identity)
+ require.Equal(t, len(want.fields), len(actual.fields), "series %q field count", identity)
+ for name, value := range want.fields {
+ require.Equal(t, value, actual.fields[name], "series %q field %q", identity, name)
+ }
+ }
+
+ // Index-rule sorted search: the previous release's SeriesSort / NIDX-03
+ // §6.2. Membership must be the same live set as the unsorted search
+ // above, and SortedValue must be non-decreasing AND must agree with
+ // each row's own (zero-padded, so byte order == numeric order) "score"
+ // field -- not just internally monotonic, but actually the right field.
+ sorted, sortedValues, sortErr := si.Search(ctx, seriesByIdentity, IndexSearchOpts{
+ Projection: []index.FieldKey{{IndexRuleID: refModelScoreRuleID}},
+ Order: &index.OrderBy{
+ Type: index.OrderByTypeIndex, Sort: modelv1.Sort_SORT_ASC,
+ Index: &databasev1.IndexRule{Metadata: &commonv1.Metadata{Id: refModelScoreRuleID}},
+ },
+ })
+ require.NoError(t, sortErr)
+ require.Len(t, sorted.SeriesList, len(model.docs), "sorted search must return the same live set")
+ require.Len(t, sortedValues, len(sorted.SeriesList))
+ scoreKey := index.FieldKey{IndexRuleID: refModelScoreRuleID}.Marshal()
+ for i := range sorted.SeriesList {
+ if i > 0 {
+ require.LessOrEqual(t, string(sortedValues[i-1]), string(sortedValues[i]), "SortedValue must be non-decreasing")
+ }
+ rowScore, ok := sorted.Fields[i][scoreKey]
+ require.True(t, ok, "sorted row %d missing its score field", i)
+ require.Equal(t, string(rowScore), string(sortedValues[i]), "SortedValue must agree with the row's own score field")
+ }
+}
+
+// TestSeriesIndex_ReferenceModel is NIDX-03 §12 item 3: a plain in-memory
+// model of the series-index contract (§2) runs against the real series
+// index -- built through newSeriesIndex, the production constructor,
+// exactly as a live segment opens one (presence cache sized like production,
+// compaction left at its conservative default, asynchronous persistence via
+// a positive flush timeout) -- under seeded random Update and external
+// receive, verified via unsorted Search, index-rule sorted Search
+// (membership + SortedValue order) and criteria filters; every observation
+// must agree. Periodic restarts (Close + reopen via newSeriesIndex) prove
+// the model's state survives exactly like a real segment reopening. The
+// seed is fixed so CI is deterministic; REFMODEL_ITERATIONS drives a longer
+// run out-of-band.
+func TestSeriesIndex_ReferenceModel(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ newSI := func() *seriesIndex {
+ si, err := newSeriesIndex(ctx, dir, 10, 4<<20, nil, &testRootLease{})
+ require.NoError(t, err)
+ return si
+ }
+ si := newSI()
+ defer func() { require.NoError(t, si.Close()) }()
+
+ rng := rand.New(rand.NewSource(1)) //nolint:gosec // deterministic test fixture, not a security context.
+ model := newRefModel()
+ identities := make([][]byte, refModelPopulation)
+ seriesByIdentity := make([]*pbv1.Series, refModelPopulation)
+ for i := 0; i < refModelPopulation; i++ {
+ identities[i], seriesByIdentity[i] = refModelSeriesIdentity(t, i)
+ }
+
+ verify := func() { verifyRefModel(ctx, t, si, seriesByIdentity, model) }
+
+ verify() // empty state: both sides agree on zero live documents.
+ iterations := referenceModelIterations()
+ // Restart roughly 4 times over the run (never on iteration 0, so the
+ // first restart has real state to survive).
+ restartEvery := iterations/4 + 1
+ for iteration := 0; iteration < iterations; iteration++ {
+ i := rng.Intn(refModelPopulation)
+ identity := identities[i]
+ timestamp := int64(iteration + 1)
+ version := int64(rng.Intn(10) + 1)
+ doc, modelDoc := buildRefModelDoc(rng, identity, timestamp, version)
+
+ if rng.Intn(2) == 0 {
+ require.NoError(t, si.Update(index.Documents{doc}))
+ model.applyUpdate(string(identity), modelDoc)
+ } else {
+ seg := buildNativeExternalSegmentMultiField(t, identity, modelDoc.fields, timestamp, version)
+ receiveExternalSegmentRetryingBusy(t, si, seg)
+ model.applyExternalReceive(string(identity), modelDoc)
+ }
+ verify()
+
+ if iteration > 0 && iteration%restartEvery == 0 {
+ // Reopen/restart (L5): a graceful Close drains everything
+ // pending (see seriesIndex.Close -> Owner.Close's final
+ // drainPersistence), so a fresh newSeriesIndex on the same
+ // directory must observe exactly the same live state.
+ require.NoError(t, si.Close())
+ si = newSI()
+ verify()
+ }
+ }
+
+ // Prefix and wildcard series matchers (L5): query group-00's three
+ // instances via a prefix matcher (trailing AnyTagValue) and instance-00
+ // across every group via a wildcard matcher (leading AnyTagValue),
+ // checking membership against the model's own live set filtered the
+ // same way.
+ t.Run("prefix matcher", func(t *testing.T) {
+ var prefixQuery pbv1.Series
+ prefixQuery.Subject = "refmodel"
+ prefixQuery.EntityValues = []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "group-00"}}},
+ pbv1.AnyTagValue,
+ }
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&prefixQuery}, IndexSearchOpts{})
+ require.NoError(t, err)
+ var want int
+ for i := 0; i < refModelInstances; i++ {
+ identity, _ := refModelSeriesIdentity(t, i) // group-00 is indices 0..refModelInstances-1.
+ if _, live := model.docs[string(identity)]; live {
+ want++
+ }
+ }
+ require.Len(t, sd.SeriesList, want, "prefix matcher must return exactly group-00's live instances")
+ for _, s := range sd.SeriesList {
+ require.Equal(t, "group-00", s.EntityValues[0].GetStr().GetValue())
+ }
+ })
+ t.Run("wildcard matcher", func(t *testing.T) {
+ var wildcardQuery pbv1.Series
+ wildcardQuery.Subject = "refmodel"
+ wildcardQuery.EntityValues = []*modelv1.TagValue{
+ pbv1.AnyTagValue,
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "instance-00"}}},
+ }
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&wildcardQuery}, IndexSearchOpts{})
+ require.NoError(t, err)
+ var want int
+ for g := 0; g < refModelGroups; g++ {
+ identity, _ := refModelSeriesIdentity(t, g*refModelInstances) // instance 0 of group g.
+ if _, live := model.docs[string(identity)]; live {
+ want++
+ }
+ }
+ require.Len(t, sd.SeriesList, want, "wildcard matcher must return exactly every group's live instance-00")
+ for _, s := range sd.SeriesList {
+ require.Equal(t, "instance-00", s.EntityValues[1].GetStr().GetValue())
+ }
+ })
+
+ // Criteria filter (L5): a tag EQ condition through the real
+ // pkg/index/native/criteria.Filter path (refModelResolver), checked
+ // against the model's own filtered subset.
+ t.Run("criteria filter", func(t *testing.T) {
+ const filterField, filterValue = "region", "v0"
+ sd, _, err := si.Search(ctx, seriesByIdentity, IndexSearchOpts{
+ Criteria: &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: filterField, Op: modelv1.Condition_BINARY_OP_EQ,
+ Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: filterValue}}},
+ }}},
+ Fields: refModelResolver{},
+ })
+ require.NoError(t, err)
+ key := index.FieldKey{TagName: filterField}.Marshal()
+ var want int
+ for _, doc := range model.docs {
+ if string(doc.fields[key]) == filterValue {
+ want++
+ }
+ }
+ require.Len(t, sd.SeriesList, want, "criteria filter must match the model's own filtered subset")
+ })
+}
+
+// TestSeriesIndex_Insert_FieldCoverageContract pins Insert's contract
+// (NIDX-03 §2.1) directly: skip a doc if its _id is live AND the segment
+// already covers every field name the new doc carries; otherwise replace.
+// It runs through the real production constructor (newSeriesIndex) but
+// stays well under the default compaction threshold (a handful of
+// admissions, default 16), so segment-wide coverage coincides with
+// per-identifier coverage and the simple assertions below are exact --
+// see TestSeriesIndex_ReferenceModel's comment on why the larger randomized
+// model avoids Insert once compaction is in play.
+func TestSeriesIndex_Insert_FieldCoverageContract(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ var series pbv1.Series
+ series.Subject = "coverage"
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "s"}}}}
+ require.NoError(t, series.Marshal())
+ identity := append([]byte(nil), series.Buffer...)
+
+ regionField := index.NewStringField(index.FieldKey{TagName: "region"}, "us")
+ regionField.Store, regionField.Index = true, true
+ require.NoError(t, si.Insert(index.Documents{{
+ Fields: []index.Field{regionField}, EntityValues: identity, Timestamp: 1,
+ }}))
+
+ // A second Insert carrying a field name already covered ("region") must
+ // be skipped: the original value survives.
+ regionField2 := index.NewStringField(index.FieldKey{TagName: "region"}, "eu")
+ regionField2.Store, regionField2.Index = true, true
+ require.NoError(t, si.Insert(index.Documents{{
+ Fields: []index.Field{regionField2}, EntityValues: identity, Timestamp: 2,
+ }}))
+
+ var queried pbv1.Series
+ queried.Subject = "coverage"
+ queried.EntityValues = series.EntityValues
+ sd, _, err := si.Search(ctx, []*pbv1.Series{&queried}, IndexSearchOpts{Projection: []index.FieldKey{{TagName: "region"}}})
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 1)
+ require.Equal(t, int64(1), sd.Timestamps[0], "a covered-field Insert must be skipped, keeping the original document")
+ require.Equal(t, []byte("us"), sd.Fields[0][index.FieldKey{TagName: "region"}.Marshal()])
+
+ // A third Insert carrying a NEW field name ("zone", not yet covered)
+ // must replace the document.
+ zoneField := index.NewStringField(index.FieldKey{TagName: "zone"}, "z1")
+ zoneField.Store, zoneField.Index = true, true
+ require.NoError(t, si.Insert(index.Documents{{
+ Fields: []index.Field{zoneField}, EntityValues: identity, Timestamp: 3,
+ }}))
+
+ sd, _, err = si.Search(ctx, []*pbv1.Series{&queried}, IndexSearchOpts{Projection: []index.FieldKey{{TagName: "zone"}}})
+ require.NoError(t, err)
+ require.Len(t, sd.SeriesList, 1)
+ require.Equal(t, int64(3), sd.Timestamps[0], "an uncovered-field Insert must replace the document")
+ require.Equal(t, []byte("z1"), sd.Fields[0][index.FieldKey{TagName: "zone"}.Marshal()])
+}
diff --git a/banyand/internal/storage/index_semantics_test.go b/banyand/internal/storage/index_semantics_test.go
new file mode 100644
index 000000000..721f93bf0
--- /dev/null
+++ b/banyand/internal/storage/index_semantics_test.go
@@ -0,0 +1,242 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query"
+)
+
+// seriesWithIndexedField builds one index.Document with an index-rule-backed
+// field (so an index order has something to sort on) plus the matching
+// *pbv1.Series to query it back.
+func seriesWithIndexedField(t *testing.T, subject string, i int, ruleID uint32, value string) (index.Document, *pbv1.Series) {
+ t.Helper()
+ var series pbv1.Series
+ series.Subject = subject
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("s%02d", i)}}}}
+ require.NoError(t, series.Marshal())
+ identity := append([]byte(nil), series.Buffer...)
+ f := index.NewStringField(index.FieldKey{IndexRuleID: ruleID}, value)
+ f.Store = true
+ f.Index = true
+ doc := index.Document{Fields: []index.Field{f}, EntityValues: identity, Timestamp: int64(i + 1)}
+ var queried pbv1.Series
+ queried.Subject = subject
+ queried.EntityValues = series.EntityValues
+ return doc, &queried
+}
+
+// TestSeriesIndex_Search_TimeOrderTakesUnsortedPath is the S1 regression: a
+// time order (resolveOrderBy's {Type: OrderByTypeTime, Index: nil}, the
+// plan's canonical shape for "no index order requested") must take the same
+// unsorted path as no order at all -- hits stay in MatchTermsSet order and
+// sortedValues is nil -- not SortHits/ProjectSortValue. Sorting on every
+// non-nil Order (the bug: `if opts.Order != nil`) changes index-mode output
+// order and flips banyand/measure/query.go's needsSorting decision.
+func TestSeriesIndex_Search_TimeOrderTakesUnsortedPath(t *testing.T) {
+ ctx := context.Background()
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ const ruleID = 77
+ var docs index.Documents
+ var queries []*pbv1.Series
+ for i := 0; i < 5; i++ {
+ doc, q := seriesWithIndexedField(t, testSubjectSvc, i, ruleID, fmt.Sprintf("v%d", i))
+ docs = append(docs, doc)
+ queries = append(queries, q)
+ }
+ require.NoError(t, si.Insert(docs))
+
+ // No order at all: the baseline unsorted shape.
+ baseline, baselineSorted, err := si.Search(ctx, queries, IndexSearchOpts{})
+ require.NoError(t, err)
+ require.Nil(t, baselineSorted)
+ require.Len(t, baseline.SeriesList, 5)
+
+ // Time order: must match the baseline's unsorted shape exactly --
+ // sortedValues nil, same hit order -- not a sorted one.
+ timeOrdered, timeOrderedSorted, err := si.Search(ctx, queries, IndexSearchOpts{Order: &index.OrderBy{Type: index.OrderByTypeTime}})
+ require.NoError(t, err)
+ require.Nil(t, timeOrderedSorted, "a time order must take the unsorted path: sortedValues must stay nil")
+ require.Len(t, timeOrdered.SeriesList, 5)
+ for i := range baseline.SeriesList {
+ require.Equal(t, baseline.SeriesList[i].EntityValues[0].GetStr().GetValue(),
+ timeOrdered.SeriesList[i].EntityValues[0].GetStr().GetValue(),
+ "a time order must not reorder hits relative to the unsorted baseline")
+ }
+
+ // Index order: by contrast, must take the sorted path.
+ indexOrdered, indexOrderedSorted, err := si.Search(ctx, queries, IndexSearchOpts{Order: &index.OrderBy{
+ Type: index.OrderByTypeIndex, Index: &databasev1.IndexRule{Metadata: &commonv1.Metadata{Id: ruleID}},
+ }})
+ require.NoError(t, err)
+ require.NotNil(t, indexOrderedSorted, "an index order must take the sorted path: sortedValues must be populated")
+ require.Len(t, indexOrderedSorted, len(indexOrdered.SeriesList))
+}
+
+// TestOrderFieldName_IndexTypeWithNilIndexErrorsNotPanics is the other half
+// of S1: {Type: OrderByTypeIndex, Index: nil} must report an error, not
+// dereference order.Index.Metadata.Id and panic.
+func TestOrderFieldName_IndexTypeWithNilIndexErrorsNotPanics(t *testing.T) {
+ require.NotPanics(t, func() {
+ _, err := orderFieldName(&index.OrderBy{Type: index.OrderByTypeIndex, Index: nil})
+ require.Error(t, err)
+ })
+}
+
+// budgetCapLease is a minimal query.BudgetLease that errors once charged
+// bytes exceed cap, for pinning that seriesIndex.search charges the query
+// memory budget on the unsorted path (S2).
+type budgetCapLease struct {
+ cap uint64
+ used uint64
+}
+
+func (l *budgetCapLease) Charge(n uint64) error {
+ l.used += n
+ if l.used > l.cap {
+ return fmt.Errorf("budget exceeded: used %d > cap %d", l.used, l.cap)
+ }
+ return nil
+}
+
+func (l *budgetCapLease) Limit() uint64 { return l.cap }
+func (l *budgetCapLease) Used() uint64 { return l.used }
+func (l *budgetCapLease) Owner() any { return nil }
+
+// TestSeriesIndex_Search_UnsortedPathChargesQueryBudget is the S2
+// regression: the previous release's parseResult/readSeriesDocument charged
+// the query memory budget per hit and per projected byte on the unsorted
+// path (stream/trace segment.Lookup, unsorted measure search); the native
+// rewrite must restore equivalent charging so a budget-exceeded error still
+// surfaces instead of search silently ignoring the lease.
+func TestSeriesIndex_Search_UnsortedPathChargesQueryBudget(t *testing.T) {
+ dir, fn := setUp(require.New(t))
+ defer fn()
+ si, err := newSeriesIndex(context.Background(), dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ var series pbv1.Series
+ series.Subject = testSubjectSvc
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "s00"}}}}
+ require.NoError(t, series.Marshal())
+ identity := append([]byte(nil), series.Buffer...)
+ require.NoError(t, si.Insert(index.Documents{{EntityValues: identity, Timestamp: 1}}))
+
+ // A cap of 1 byte is exhausted by the very first (fixed, 1024-byte
+ // floor) charge in search(), regardless of the identifier's actual
+ // length -- proving charging happens at all, not any particular amount.
+ lease := &budgetCapLease{cap: 1}
+ ctx := query.WithBudgetLease(context.Background(), lease)
+
+ var queried pbv1.Series
+ queried.Subject = testSubjectSvc
+ queried.EntityValues = series.EntityValues
+ _, _, searchErr := si.Search(ctx, []*pbv1.Series{&queried}, IndexSearchOpts{})
+ require.Error(t, searchErr, "an exhausted query budget must surface as an error from the unsorted search path")
+ require.Greater(t, lease.used, uint64(0), "search must have charged the budget before erroring")
+}
+
+// TestEncodeSeriesDocument_UnindexedFieldAlwaysStored is the S5 regression:
+// an Index=false field must always be stored (the previous release's
+// bluge.NewStoredOnlyField, NIDX-03 §6.1), regardless of the caller's Store
+// value -- an Index=false, Store=false field must not silently carry
+// neither an index entry nor a stored value.
+func TestEncodeSeriesDocument_UnindexedFieldAlwaysStored(t *testing.T) {
+ f := index.NewStringField(index.FieldKey{TagName: "unindexed"}, "value")
+ f.Index = false
+ f.Store = false // the caller under-declares Store; EncodeSeriesDocument must correct it.
+
+ var series pbv1.Series
+ series.Subject = testSubjectSvc
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "s00"}}}}
+ require.NoError(t, series.Marshal())
+
+ doc := index.Document{Fields: []index.Field{f}, EntityValues: append([]byte(nil), series.Buffer...), Timestamp: 1}
+ nd, err := EncodeSeriesDocument(doc)
+ require.NoError(t, err)
+ require.Len(t, nd.Fields, 1)
+ require.True(t, nd.Fields[0].Store, "an Index=false field must always be stored, regardless of the caller's Store value")
+ require.False(t, nd.Fields[0].Index)
+}
+
+// TestRemoveLegacySeriesIndexArtifacts_TargetsSiblingExternalSegmentTempDir
+// is the L3 regression: the previous release opened its bluge store with
+// ExternalSegmentTempDir: path.Join(root, ...) where root is the SEGMENT
+// directory -- a SIBLING of root/sidx, not a subdirectory of it. A fresh
+// newSeriesIndex must remove both the legacy lock file (inside sidx) and the
+// legacy external-segment-temp directory at its real, sibling location, and
+// must never panic if either happens to be unremovable.
+func TestRemoveLegacySeriesIndexArtifacts_TargetsSiblingExternalSegmentTempDir(t *testing.T) {
+ ctx := context.Background()
+ root, fn := setUp(require.New(t))
+ defer fn()
+
+ indexPath := filepath.Join(root, seriesIndexDirName)
+ require.NoError(t, os.MkdirAll(indexPath, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(indexPath, legacyLockFilename), []byte("pid"), 0o600))
+
+ legacyExternalSegmentTempDir := filepath.Join(root, legacyExternalSegmentTempDirName)
+ require.NoError(t, os.MkdirAll(legacyExternalSegmentTempDir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(legacyExternalSegmentTempDir, "staged.bin"), []byte("x"), 0o600))
+
+ // The wrong (pre-fix) location: root/sidx/external-segment-temp. It must
+ // not exist in this fixture, and newSeriesIndex must not need it to
+ // exist to clean up the real, sibling one.
+ wrongLocation := filepath.Join(indexPath, legacyExternalSegmentTempDirName)
+ _, statErr := os.Stat(wrongLocation)
+ require.True(t, os.IsNotExist(statErr))
+
+ si, err := newSeriesIndex(ctx, root, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, si.Close()) }()
+
+ _, lockStatErr := os.Stat(filepath.Join(indexPath, legacyLockFilename))
+ require.True(t, os.IsNotExist(lockStatErr), "the legacy lock file must be removed")
+ _, dirStatErr := os.Stat(legacyExternalSegmentTempDir)
+ require.True(t, os.IsNotExist(dirStatErr), "the legacy external-segment-temp directory at its real, sibling location must be removed")
+}
+
+// TestNewOwner_RejectsNilLease documents the L9 contract
+// newSegmentController's production path now relies on instead of a
+// silently-substituted default lease: pkg/index/native.NewOwner itself
+// refuses a nil lease with ErrLeaseUnavailable.
+func TestNewOwner_RejectsNilLease(t *testing.T) {
+ dir := t.TempDir()
+ _, err := native.NewOwner(native.OwnerOptions{Path: filepath.Join(dir, "sidx"), IdentifierDocValues: true})
+ require.ErrorIs(t, err, native.ErrLeaseUnavailable)
+}
diff --git a/banyand/internal/storage/index_test.go b/banyand/internal/storage/index_test.go
index 143e7ee5c..b01143b65 100644
--- a/banyand/internal/storage/index_test.go
+++ b/banyand/internal/storage/index_test.go
@@ -34,10 +34,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/test/flags"
)
+// testSubjectSvc is the fixed series Subject several series-index tests
+// across this package use for synthetic fixtures.
+const testSubjectSvc = "svc"
+
func TestSeriesIndex_Primary(t *testing.T) {
ctx := context.Background()
path, fn := setUp(require.New(t))
- si, err := newSeriesIndex(ctx, path, 0, 0, nil)
+ si, err := newSeriesIndex(ctx, path, 0, 0, nil, &testRootLease{})
require.NoError(t, err)
defer func() {
require.NoError(t, si.Close())
@@ -67,7 +71,7 @@ func TestSeriesIndex_Primary(t *testing.T) {
require.NoError(t, si.Insert(docs))
// Restart the index
require.NoError(t, si.Close())
- si, err = newSeriesIndex(ctx, path, 0, 0, nil)
+ si, err = newSeriesIndex(ctx, path, 0, 0, nil, &testRootLease{})
require.NoError(t, err)
tests := []struct {
name string
@@ -158,7 +162,7 @@ func TestSeriesIndex_Primary(t *testing.T) {
seriesQuery.EntityValues = tt.entityValues[i]
seriesQueries = append(seriesQueries, &seriesQuery)
}
- sd, err := si.filter(ctx, seriesQueries, nil, nil, nil)
+ sd, err := si.filter(ctx, seriesQueries)
sl := sd.SeriesList
require.NoError(t, err)
require.Equal(t, len(tt.entityValues), len(sl))
diff --git a/banyand/internal/storage/nidx03_benchmark_test.go b/banyand/internal/storage/nidx03_benchmark_test.go
new file mode 100644
index 000000000..7da891706
--- /dev/null
+++ b/banyand/internal/storage/nidx03_benchmark_test.go
@@ -0,0 +1,541 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "testing"
+ "time"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// NIDX-03 §13 performance benchmark. It measures the IndexDB operations the
+// design's performance table names (write hot path, exact/prefix/wildcard
+// lookup, index-order Search, external receive) over banyand/internal/storage's
+// series index, built with newSeriesIndex exactly as a live segment would.
+//
+// Scale: nidx03BenchSeriesCount series, split into nidx03BenchGroups groups of
+// nidx03BenchInstancesPerGroup instances each (default 1000x1000 = 1,000,000,
+// per §13's own target; override with the NIDX03_BENCH_SERIES_COUNT env var
+// to a smaller perfect square -- e.g. 40000 for a 200x200 grid -- if 1M proves
+// impractical in a given environment). Fixture construction (bulk Insert) is
+// unavoidably expensive at this scale; every benchmark pays it once in its own
+// setup, outside the timed region, and it dominates `-count>1` wall time more
+// than the operation under measurement -- this is an accepted cost of
+// benchmarking at the design's stated scale, not a flaw in the harness.
+//
+// Porting to the 735e9ad2 (previous-release) tree changes exactly three
+// functions (every Benchmark function and nidx03BenchFixture method below is
+// otherwise unchanged, because 735e9ad2's seriesIndex.Insert/Search/
+// EnableExternalSegments and IndexSearchOpts{Order, Projection} already have
+// this shape):
+// - buildBenchSeriesIndex: its newSeriesIndex call drops the RootLease
+// argument, and metrics is *inverted.Metrics instead of *metrics.Metrics.
+// - nidx03BenchTakeFileSnapshot: 735e9ad2's IndexDB has no TakeFileSnapshot
+// method (added in NIDX-03 phase 2); the ported copy calls
+// si.store.TakeFileSnapshot(dir) instead.
+// - nidx03SettleCompaction: a no-op in the ported copy. Here it brings the
+// native owner to a quiescent state (tiered plan exhausted, root
+// durable) so no background merge or persist overlaps a timed region;
+// the retired bluge engine has no equivalent Compact call to drive.
+//
+// Methodology: every operation benchmark closes its fixture in a b.Cleanup
+// (outside the timed region), runs the operation once untimed (reported as
+// the first-op-ns metric, the one-time lazy per-segment initialization cost)
+// before resetting the timer, and should be run with an explicit iteration
+// count (-benchtime=Nx, N > 1) so each reported ns/op averages N warm
+// operations instead of one cold sample.
+const (
+ nidx03BenchGroupsDefault = 1000
+ nidx03BenchInstancesPerGroupDefault = 1000
+ nidx03BenchSubject = "nidx03_bench"
+)
+
+// nidx03BenchCounts resolves (groups, instancesPerGroup) from
+// NIDX03_BENCH_SERIES_COUNT (a perfect square; default 1_000_000 ->
+// 1000x1000), so a constrained environment can run this at a smaller, still
+// square, scale without editing the file.
+func nidx03BenchCounts(b *testing.B) (groups, perGroup int) {
+ b.Helper()
+ groups, perGroup = nidx03BenchGroupsDefault, nidx03BenchInstancesPerGroupDefault
+ raw := os.Getenv("NIDX03_BENCH_SERIES_COUNT")
+ if raw == "" {
+ return groups, perGroup
+ }
+ n, err := strconv.Atoi(raw)
+ if err != nil || n <= 0 {
+ b.Fatalf("NIDX03_BENCH_SERIES_COUNT=%q must be a positive integer", raw)
+ }
+ root := isqrt(n)
+ if root*root != n {
+ b.Fatalf("NIDX03_BENCH_SERIES_COUNT=%d must be a perfect square (groups == instancesPerGroup)", n)
+ }
+ return root, root
+}
+
+func isqrt(n int) int {
+ if n <= 0 {
+ return 0
+ }
+ x := n
+ y := (x + 1) / 2
+ for y < x {
+ x = y
+ y = (x + n/x) / 2
+ }
+ return x
+}
+
+// nidx03BenchFixture owns the on-disk series index every benchmark below
+// opens fresh (via b.TempDir(), never shared across Benchmark functions, so
+// one benchmark's writes never skew another's measurement) and the exact
+// series identities it was built from, so operation benchmarks can look one
+// up without recomputing the marshaled form on every b.N iteration.
+type nidx03BenchFixture struct {
+ si *seriesIndex
+ groupOf func(i int) string
+ instanceOf func(i int) string
+ seriesCount int
+ groups int
+ perGroup int
+}
+
+// buildBenchSeriesIndex is one of the two functions a 735e9ad2 port of this
+// file changes: newSeriesIndex there takes no RootLease argument and metrics
+// is *inverted.Metrics (the 735e9ad2 copy drops ", &testRootLease{}" from
+// the newSeriesIndex call below). The other is nidx03BenchTakeFileSnapshot.
+func buildBenchSeriesIndex(b *testing.B) (*seriesIndex, string) {
+ b.Helper()
+ root := b.TempDir()
+ si, err := newSeriesIndex(context.Background(), root, 0, 0, nil, &testRootLease{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ // Close runs as a cleanup, after the benchmark harness has stopped the
+ // timer. A `defer si.Close()` in a Benchmark function runs before the
+ // function returns, i.e. inside the timed region, and with -benchtime=1x
+ // Close (the native owner's final drain, waiting out an in-flight
+ // background merge) dominated every row's measurement. Callers must not
+ // Close si themselves: the previous release's Close is not idempotent.
+ b.Cleanup(func() { _ = si.Close() })
+ return si, filepath.Join(root, "sidx")
+}
+
+// nidx03BenchTakeFileSnapshot is the second (and last) constructor-shaped
+// difference a 735e9ad2 port of this file changes: IndexDB there (added in
+// NIDX-03 phase 2, before this phase 3 change) has no TakeFileSnapshot
+// method, so the ported copy opens si's package-private bluge-backed store
+// field directly -- `return si.store.TakeFileSnapshot(dir)` -- instead of
+// calling through seriesIndex.
+func nidx03BenchTakeFileSnapshot(si *seriesIndex, dir string) error {
+ return si.TakeFileSnapshot(dir)
+}
+
+// nidx03SettleCompaction brings the fixture to a quiescent, steady state
+// before any timed region: it drives the owner's bounded-merge Compact until
+// the tiered plan is a no-op (the root generation stops advancing), then
+// waits until that root is durable. A plan that is a no-op on the current
+// root leaves the owner's background maintenance goroutine nothing to merge
+// either, and waiting for durability also waits out persistence's handle
+// promotion, so neither a background merge nor a persist overlaps the timed
+// region. A lost race with background maintenance (ErrStaleCompaction) or a
+// concurrent collection (ErrPersistenceBusy) is retried, not treated as
+// "settled".
+func nidx03SettleCompaction(b *testing.B, si *seriesIndex, _ string) {
+ b.Helper()
+ ctx := context.Background()
+ generation := func() uint64 {
+ view, err := si.owner.Acquire(ctx)
+ if err != nil {
+ b.Fatal(err)
+ }
+ g := view.Generation()
+ _ = view.Close()
+ return g
+ }
+ settled := false
+ for i := 0; i < 1000 && !settled; i++ {
+ before := generation()
+ err := si.owner.Compact(ctx)
+ switch {
+ case errors.Is(err, native.ErrStaleCompaction), errors.Is(err, native.ErrPersistenceBusy):
+ time.Sleep(10 * time.Millisecond)
+ case err != nil:
+ b.Fatal(err)
+ default:
+ settled = generation() == before
+ }
+ }
+ if !settled {
+ b.Fatal("series index compaction did not settle")
+ }
+ deadline := time.Now().Add(5 * time.Minute)
+ for si.owner.DurableGeneration() < generation() {
+ if time.Now().After(deadline) {
+ b.Fatal("series index did not become durable")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ // Collect superseded segment files now rather than letting the
+ // post-persist collection wake run inside a timed region.
+ for i := 0; i < 100; i++ {
+ err := si.owner.CollectGarbage(ctx)
+ if !errors.Is(err, native.ErrPersistenceBusy) {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+// nidx03WarmUp runs op once outside the timed region, then resets the
+// timer. It returns a function the caller must invoke after its timed loop
+// to report that first call's latency as the "first-op-ns" metric (a metric
+// reported before ResetTimer would be discarded by it). The first
+// operation after a segment is published or promoted pays one-time lazy
+// per-segment initialization (term dictionary load, bloom filter build,
+// stored-document offset table), which is real but is a different quantity
+// from the steady-state per-op cost the §13 rows compare.
+func nidx03WarmUp(b *testing.B, op func()) (reportFirstOp func()) {
+ b.Helper()
+ start := time.Now()
+ op()
+ first := time.Since(start)
+ b.ReportAllocs()
+ b.ResetTimer()
+ return func() { b.ReportMetric(float64(first.Nanoseconds()), "first-op-ns") }
+}
+
+func nidx03BenchSeries(group, instance string) *pbv1.Series {
+ return &pbv1.Series{
+ Subject: nidx03BenchSubject,
+ EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: group}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: instance}}},
+ },
+ }
+}
+
+const nidx03BenchScoreRuleID = 7
+
+// buildNidx03BenchFixture inserts groups*perGroup series (each carrying an
+// indexed, sortable "score" field so index-order Search has something to
+// sort on) into a fresh series index, batching the writes so construction at
+// 1M series completes in a practical amount of wall time.
+func buildNidx03BenchFixture(b *testing.B) *nidx03BenchFixture {
+ b.Helper()
+ groups, perGroup := nidx03BenchCounts(b)
+ si, sidxPath := buildBenchSeriesIndex(b)
+
+ const batchSize = 5000
+ var batch index.Documents
+ flush := func() {
+ if len(batch) == 0 {
+ return
+ }
+ if err := si.Insert(batch); err != nil {
+ b.Fatal(err)
+ }
+ batch = batch[:0]
+ }
+ scoreKey := index.FieldKey{IndexRuleID: nidx03BenchScoreRuleID}
+ n := 0
+ for g := 0; g < groups; g++ {
+ group := fmt.Sprintf("group-%06d", g)
+ for i := 0; i < perGroup; i++ {
+ instance := fmt.Sprintf("instance-%06d", i)
+ series := nidx03BenchSeries(group, instance)
+ if err := series.Marshal(); err != nil {
+ b.Fatal(err)
+ }
+ score := index.NewBytesField(scoreKey, convert.Int64ToBytes(int64(n)))
+ score.Store, score.Index = true, true
+ batch = append(batch, index.Document{
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Fields: []index.Field{score},
+ Timestamp: int64(n + 1),
+ })
+ n++
+ if len(batch) >= batchSize {
+ flush()
+ }
+ }
+ }
+ flush()
+ nidx03SettleCompaction(b, si, sidxPath)
+ return &nidx03BenchFixture{
+ si: si,
+ groups: groups,
+ perGroup: perGroup,
+ seriesCount: n,
+ groupOf: func(i int) string { return fmt.Sprintf("group-%06d", i) },
+ instanceOf: func(i int) string { return fmt.Sprintf("instance-%06d", i) },
+ }
+}
+
+// BenchmarkSeriesIndexInsertExisting measures the write hot path NIDX-03 §13
+// names: Insert of series that already exist with every field the incoming
+// document carries, which InsertIfAbsent admission must recognize and skip
+// without encoding a new document.
+func BenchmarkSeriesIndexInsertExisting(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+
+ const reinsertBatch = 100
+ docs := make(index.Documents, 0, reinsertBatch)
+ scoreKey := index.FieldKey{IndexRuleID: nidx03BenchScoreRuleID}
+ for i := 0; i < reinsertBatch; i++ {
+ series := nidx03BenchSeries(fixture.groupOf(0), fixture.instanceOf(i))
+ if err := series.Marshal(); err != nil {
+ b.Fatal(err)
+ }
+ score := index.NewBytesField(scoreKey, convert.Int64ToBytes(int64(i)))
+ score.Store, score.Index = true, true
+ docs = append(docs, index.Document{
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Fields: []index.Field{score},
+ Timestamp: int64(i + 1),
+ })
+ }
+
+ insert := func() {
+ if err := fixture.si.Insert(docs); err != nil {
+ b.Fatal(err)
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, insert)
+ for i := 0; i < b.N; i++ {
+ insert()
+ }
+ reportFirstOp()
+}
+
+// BenchmarkSeriesIndexLookupExact measures Search with a single exact `_id`
+// series matcher, no order, no criteria -- Stream/Trace segment.Lookup's
+// shape.
+func BenchmarkSeriesIndexLookupExact(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+ ctx := context.Background()
+ target := []*pbv1.Series{nidx03BenchSeries(fixture.groupOf(fixture.groups/2), fixture.instanceOf(fixture.perGroup/2))}
+
+ lookup := func() {
+ sd, _, err := fixture.si.Search(ctx, target, IndexSearchOpts{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(sd.SeriesList) != 1 {
+ b.Fatalf("expected exactly one exact match, got %d", len(sd.SeriesList))
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, lookup)
+ for i := 0; i < b.N; i++ {
+ lookup()
+ }
+ reportFirstOp()
+}
+
+// BenchmarkSeriesIndexLookupPrefix measures Search with a prefix series
+// matcher (trailing AnyTagValue): one group's perGroup instances.
+func BenchmarkSeriesIndexLookupPrefix(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+ ctx := context.Background()
+ prefixQuery := &pbv1.Series{
+ Subject: nidx03BenchSubject,
+ EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fixture.groupOf(fixture.groups / 2)}}},
+ pbv1.AnyTagValue,
+ },
+ }
+
+ lookup := func() {
+ sd, _, err := fixture.si.Search(ctx, []*pbv1.Series{prefixQuery}, IndexSearchOpts{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(sd.SeriesList) != fixture.perGroup {
+ b.Fatalf("expected %d prefix matches, got %d", fixture.perGroup, len(sd.SeriesList))
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, lookup)
+ for i := 0; i < b.N; i++ {
+ lookup()
+ }
+ reportFirstOp()
+}
+
+// BenchmarkSeriesIndexLookupWildcard measures Search with a wildcard series
+// matcher (leading AnyTagValue): one instance across every group.
+func BenchmarkSeriesIndexLookupWildcard(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+ ctx := context.Background()
+ wildcardQuery := &pbv1.Series{
+ Subject: nidx03BenchSubject,
+ EntityValues: []*modelv1.TagValue{
+ pbv1.AnyTagValue,
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fixture.instanceOf(fixture.perGroup / 2)}}},
+ },
+ }
+
+ lookup := func() {
+ sd, _, err := fixture.si.Search(ctx, []*pbv1.Series{wildcardQuery}, IndexSearchOpts{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(sd.SeriesList) != fixture.groups {
+ b.Fatalf("expected %d wildcard matches, got %d", fixture.groups, len(sd.SeriesList))
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, lookup)
+ for i := 0; i < b.N; i++ {
+ lookup()
+ }
+ reportFirstOp()
+}
+
+// BenchmarkSeriesIndexSearchIndexOrder measures Search with an index-rule
+// sort order over one group's instances -- the Measure index-rule sort path
+// (view.SortHits / ProjectSortValue).
+func BenchmarkSeriesIndexSearchIndexOrder(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+ ctx := context.Background()
+ prefixQuery := &pbv1.Series{
+ Subject: nidx03BenchSubject,
+ EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fixture.groupOf(fixture.groups / 2)}}},
+ pbv1.AnyTagValue,
+ },
+ }
+ order := &index.OrderBy{
+ Type: index.OrderByTypeIndex,
+ Sort: modelv1.Sort_SORT_ASC,
+ Index: &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Id: nidx03BenchScoreRuleID},
+ },
+ }
+
+ search := func() {
+ // PreloadSize is unused by the current engine's Search but is the
+ // previous release's bluge sortIterator page size: a zero value
+ // starves it to zero results there, so every benchmark in this file
+ // sets it explicitly to keep both trees' call shape identical.
+ sd, sortedValues, err := fixture.si.Search(ctx, []*pbv1.Series{prefixQuery}, IndexSearchOpts{Order: order, PreloadSize: fixture.perGroup})
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(sd.SeriesList) != fixture.perGroup || len(sortedValues) != fixture.perGroup {
+ b.Fatalf("expected %d sorted matches, got %d series / %d sort values", fixture.perGroup, len(sd.SeriesList), len(sortedValues))
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, search)
+ for i := 0; i < b.N; i++ {
+ search()
+ }
+ reportFirstOp()
+}
+
+// BenchmarkSeriesIndexExternalReceive measures EnableExternalSegments'
+// StartSegment/WriteChunk/CompleteSegment cycle: the crash-safe raw segment
+// receive path the lifecycle/*-series-sync receivers drive.
+func BenchmarkSeriesIndexExternalReceive(b *testing.B) {
+ fixture := buildNidx03BenchFixture(b)
+
+ // Build one committed segment's bytes per iteration (plus one for the
+ // warm-up), each from a fresh, separate series index under its own
+ // subject, so every received segment is wholly new to fixture.si: the
+ // common receive case this benchmark targets, not the keep-existing
+ // dedup path index_external_segments_test.go already covers. Reusing
+ // one payload would turn every iteration after the first into a fully
+ // deduplicated receive once b.N > 1.
+ payloads := make([][]byte, b.N+1)
+ for i := range payloads {
+ payloads[i] = nidx03BenchExternalSegmentPayload(b, i)
+ }
+ receive := func(payload []byte) {
+ streamer, err := fixture.si.EnableExternalSegments()
+ if err != nil {
+ b.Fatal(err)
+ }
+ if err := streamer.StartSegment(); err != nil {
+ b.Fatal(err)
+ }
+ if err := streamer.WriteChunk(payload); err != nil {
+ b.Fatal(err)
+ }
+ if err := streamer.CompleteSegment(); err != nil {
+ b.Fatal(err)
+ }
+ }
+ reportFirstOp := nidx03WarmUp(b, func() { receive(payloads[b.N]) })
+ for i := 0; i < b.N; i++ {
+ receive(payloads[i])
+ }
+ reportFirstOp()
+}
+
+// nidx03BenchExternalSegmentPayload builds one committed segment file's raw
+// bytes from a fresh, separate series index: 5000 documents under a subject
+// (distinct per ordinal) no fixture series or other payload uses, so
+// EnableExternalSegments admits every document as new.
+func nidx03BenchExternalSegmentPayload(b *testing.B, ordinal int) []byte {
+ b.Helper()
+ const externalDocs = 5000
+ // Closed by buildBenchSeriesIndex's cleanup; the previous release's
+ // Close is not idempotent, so no second, explicit Close here.
+ si, _ := buildBenchSeriesIndex(b)
+ var docs index.Documents
+ for i := 0; i < externalDocs; i++ {
+ series := &pbv1.Series{
+ Subject: fmt.Sprintf("nidx03_bench_external_%d", ordinal),
+ EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("ext-%06d", i)}}},
+ },
+ }
+ if err := series.Marshal(); err != nil {
+ b.Fatal(err)
+ }
+ docs = append(docs, index.Document{EntityValues: append([]byte(nil), series.Buffer...), Timestamp: int64(i + 1)})
+ }
+ if err := si.Insert(docs); err != nil {
+ b.Fatal(err)
+ }
+ segDir := b.TempDir()
+ if err := nidx03BenchTakeFileSnapshot(si, segDir); err != nil {
+ b.Fatal(err)
+ }
+ matches, globErr := filepath.Glob(filepath.Join(segDir, "*.seg"))
+ if globErr != nil {
+ b.Fatal(globErr)
+ }
+ if len(matches) != 1 {
+ b.Fatalf("expected exactly one segment file in %s, found %d", segDir, len(matches))
+ }
+ data, readErr := os.ReadFile(matches[0])
+ if readErr != nil {
+ b.Fatal(readErr)
+ }
+ return data
+}
diff --git a/banyand/internal/storage/nidx03_dependency_guard_test.go b/banyand/internal/storage/nidx03_dependency_guard_test.go
new file mode 100644
index 000000000..c23a7b245
--- /dev/null
+++ b/banyand/internal/storage/nidx03_dependency_guard_test.go
@@ -0,0 +1,334 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// nidx03RetiredSeriesSymbols are the pkg/index/inverted exported names NIDX-03
+// §10 removed completely: the legacy series-index query builders
+// (BuildIndexModeQuery -- and its unexported helper buildIndexModeCriteria)
+// and the Property query builders that moved to the native criteria filter
+// alongside them, since nothing else kept query.go's node-building
+// apparatus alive once they were gone, per NIDX-03 §15. The series index
+// itself has used native.Owner exclusively since NIDX-03 phase 2; this test
+// is phase 3's companion proof that those symbols are actually gone, not
+// just unreferenced.
+//
+// BuildQuery is deliberately NOT in this list: index.SeriesStore's interface
+// still requires the method to exist on *store (the NIDX-04 element
+// migration tool's only remaining dependency), so §15's "reduce to the
+// minimum NIDX-04 needs" leaves it declared as an always-failing stub
+// instead of removing it. TestRetiredBuildQueryIsAnAlwaysFailingStub checks
+// that stub shape directly, so a regression that made BuildQuery do real
+// query-building work again is still caught, just not by this list.
+var nidx03RetiredSeriesSymbols = []string{
+ "BuildIndexModeQuery",
+ "buildIndexModeCriteria",
+ "BuildPropertyQuery",
+ "BuildPropertyQueryFromEntity",
+}
+
+// nidx03AllowedStreamInvertedImporters are the only banyand/stream
+// non-test .go files NIDX-03 permits to import pkg/index/inverted: the
+// Stream element index migration tool NIDX-04 (not this change) will remove,
+// per the design doc's explicit scope boundary ("Out of scope: the rest of
+// the Stream element idx work and its migration tool"). Every other
+// banyand/stream series- or element-index file reaches the native engine
+// directly or through pkg/index/nativeadapter.
+var nidx03AllowedStreamInvertedImporters = []string{"migration_element_index.go"}
+
+// nidx03AllowedStreamBlugeImporters are the only banyand/stream non-test .go
+// files NIDX-03 permits to import blugelabs/bluge directly (not through
+// pkg/index/inverted): migration_verify.go's CountBlugeDocs, which opens the
+// previous-release, bluge-format Stream element index (idx/) directory
+// read-only to compare document counts against the native-written copy --
+// also NIDX-04 element-index scope, like migration_element_index.go, just a
+// raw reader instead of a full writer/searcher.
+var nidx03AllowedStreamBlugeImporters = []string{"migration_verify.go"}
+
+// nidx03AllowedStreamTransitiveBlugeOrInvertedDeps is banyand/stream's full
+// `go list -deps` transitive closure of every package named
+// "blugelabs/bluge*" or "pkg/index/inverted" -- the whole dependency
+// subgraph nidx03AllowedStreamInvertedImporters and
+// nidx03AllowedStreamBlugeImporters's two files pull in, not just the two
+// packages they import directly. TestStreamTransitiveDepsReachOnlyTheAllowedBlugeSubgraph
+// asserts this is the complete set: a new, unrelated banyand/stream
+// dependency that happens to reach bluge or pkg/index/inverted through some
+// OTHER path (not migration_element_index.go or migration_verify.go) would
+// add an entry here this test doesn't expect, and fail.
+var nidx03AllowedStreamTransitiveBlugeOrInvertedDeps = []string{
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted",
+ "github.com/blugelabs/bluge",
+ "github.com/blugelabs/bluge/analysis",
+ "github.com/blugelabs/bluge/analysis/analyzer",
+ "github.com/blugelabs/bluge/analysis/lang/en",
+ "github.com/blugelabs/bluge/analysis/token",
+ "github.com/blugelabs/bluge/analysis/tokenizer",
+ "github.com/blugelabs/bluge/index",
+ "github.com/blugelabs/bluge/index/lock",
+ "github.com/blugelabs/bluge/index/mergeplan",
+ "github.com/blugelabs/bluge/numeric",
+ "github.com/blugelabs/bluge/numeric/geo",
+ "github.com/blugelabs/bluge/search",
+ "github.com/blugelabs/bluge/search/aggregations",
+ "github.com/blugelabs/bluge/search/collector",
+ "github.com/blugelabs/bluge/search/searcher",
+ "github.com/blugelabs/bluge/search/similarity",
+ "github.com/blugelabs/bluge_segment_api",
+ "github.com/blugelabs/ice",
+ "github.com/blugelabs/ice/compress",
+}
+
+// repositoryRoot locates the module root from this test file's own path, so
+// the guard runs correctly regardless of the working directory `go test`
+// was invoked from.
+func repositoryRoot(t *testing.T) string {
+ t.Helper()
+ _, sourceFile, _, ok := runtime.Caller(0)
+ require.True(t, ok)
+ // This file lives at <root>/banyand/internal/storage/<this file>.
+ return filepath.Join(filepath.Dir(sourceFile), "..", "..", "..")
+}
+
+// TestSeriesIndexConsumersReachNoRetiredSeriesCode is the NIDX-03 §12 item 8
+// / merge-proof guard: "go list -deps of the storage, measure, trace, and
+// measure-query-planner packages reaches no series code in
+// pkg/index/inverted [and no github.com/blugelabs/bluge]". banyand/stream is
+// checked separately (TestStreamOnlyImportsInvertedThroughElementMigrationTool)
+// because it legitimately still depends on pkg/index/inverted for the
+// NIDX-04 element-index migration tool.
+func TestSeriesIndexConsumersReachNoRetiredSeriesCode(t *testing.T) {
+ root := repositoryRoot(t)
+ targets := []string{
+ "./banyand/internal/storage",
+ "./banyand/measure",
+ "./banyand/trace",
+ "./pkg/query/vectorized/measure/plan",
+ }
+ for _, target := range targets {
+ t.Run(target, func(t *testing.T) {
+ deps := goListDeps(t, root, target)
+ for _, dependency := range deps {
+ require.NotContains(t, dependency, "/pkg/index/inverted",
+ "%s must not reach the retired series-index package through %s", target, dependency)
+ if dependency == "github.com/blugelabs/bluge/numeric" {
+ // Not the retired engine: a standalone prefix-coded-numeric
+ // encoding helper banyand/internal/sidx's tag filter range
+ // operator has used since #991 (long before NIDX-01/02/03),
+ // for a purpose unrelated to the series index or
+ // pkg/index/inverted. It has no import edge back to the
+ // bluge engine (bluge, bluge/index, bluge/search,
+ // bluge_segment_api) this guard retires; every other
+ // "blugelabs/bluge" dependency is still checked below.
+ continue
+ }
+ require.NotContains(t, dependency, "github.com/blugelabs/bluge",
+ "%s must not reach the retired bluge engine through %s", target, dependency)
+ }
+ })
+ }
+}
+
+// TestStreamOnlyImportsInvertedThroughElementMigrationTool is the
+// banyand/stream half of the same merge proof: its one legitimate
+// pkg/index/inverted dependency -- the NIDX-04 element-index migration tool,
+// out of this change's scope -- is exactly who the package's production
+// (non-test) source reaches it through. Any other file importing
+// pkg/index/inverted is either a NIDX-03 regression (series-index code that
+// should have moved to the native engine) or a new, undocumented exception
+// this guard is designed to catch.
+func TestStreamOnlyImportsInvertedThroughElementMigrationTool(t *testing.T) {
+ root := repositoryRoot(t)
+ invertedImporters, blugeImporters := nidx03StreamDirectImporters(t, root)
+ require.Equal(t, nidx03AllowedStreamInvertedImporters, invertedImporters,
+ "banyand/stream's non-test pkg/index/inverted importers must be exactly the NIDX-04 element migration tool")
+ require.Equal(t, nidx03AllowedStreamBlugeImporters, blugeImporters,
+ "banyand/stream's non-test direct blugelabs/bluge importers must be exactly the NIDX-04 element-index verify tool")
+}
+
+// TestStreamTransitiveDepsReachOnlyTheAllowedBlugeSubgraph is R6's
+// "transitive deps, not just direct imports" half: TestStreamOnlyImports...
+// above only looks at banyand/stream's OWN files' import statements, which
+// proves migration_element_index.go and migration_verify.go are the direct
+// edges into bluge/pkg-index-inverted, but says nothing about whether some
+// OTHER, unrelated banyand/stream dependency also reaches that subgraph
+// through a different path entirely (which the direct-import check can't
+// see, since it never looks past banyand/stream's own files). `go list
+// -deps` sees the whole transitive closure regardless of which file causes
+// which edge, so comparing its bluge/inverted-shaped slice against the
+// frozen expected set catches that case specifically.
+func TestStreamTransitiveDepsReachOnlyTheAllowedBlugeSubgraph(t *testing.T) {
+ root := repositoryRoot(t)
+ deps := goListDeps(t, root, "./banyand/stream")
+ var found []string
+ for _, dependency := range deps {
+ if strings.Contains(dependency, "blugelabs/bluge") || strings.Contains(dependency, "blugelabs/ice") ||
+ dependency == "github.com/apache/skywalking-banyandb/pkg/index/inverted" {
+ found = append(found, dependency)
+ }
+ }
+ sort.Strings(found)
+ expected := append([]string(nil), nidx03AllowedStreamTransitiveBlugeOrInvertedDeps...)
+ sort.Strings(expected)
+ require.Equal(t, expected, found,
+ "banyand/stream's transitive bluge/pkg-index-inverted dependency subgraph must be exactly the one "+
+ "migration_element_index.go and migration_verify.go are known to pull in")
+}
+
+// nidx03StreamDirectImporters parses every banyand/stream non-test .go
+// file's own imports and returns, separately, the file names (sorted) that
+// directly import pkg/index/inverted and those that directly import
+// blugelabs/bluge.
+func nidx03StreamDirectImporters(t *testing.T, root string) (invertedImporters, blugeImporters []string) {
+ t.Helper()
+ streamDir := filepath.Join(root, "banyand", "stream")
+ entries, err := os.ReadDir(streamDir)
+ require.NoError(t, err)
+
+ fileSet := token.NewFileSet()
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ path := filepath.Join(streamDir, name)
+ file, parseErr := parser.ParseFile(fileSet, path, nil, parser.ImportsOnly)
+ require.NoError(t, parseErr, "parse %s", path)
+ for _, imported := range file.Imports {
+ importPath := strings.Trim(imported.Path.Value, `"`)
+ switch {
+ case importPath == "github.com/apache/skywalking-banyandb/pkg/index/inverted":
+ invertedImporters = append(invertedImporters, name)
+ case importPath == "github.com/blugelabs/bluge":
+ blugeImporters = append(blugeImporters, name)
+ }
+ }
+ }
+ sort.Strings(invertedImporters)
+ sort.Strings(blugeImporters)
+ return invertedImporters, blugeImporters
+}
+
+// TestRetiredBuildQueryIsAnAlwaysFailingStub is BuildQuery's own check, kept
+// separate from nidx03RetiredSeriesSymbols (see that var's comment): it
+// confirms the method still exists (index.SeriesStore requires it) but does
+// no real query-building work -- it always returns an error -- so a
+// regression that quietly restored its old behavior is still caught.
+func TestRetiredBuildQueryIsAnAlwaysFailingStub(t *testing.T) {
+ root := repositoryRoot(t)
+ invertedDir := filepath.Join(root, "pkg", "index", "inverted")
+ fileSet := token.NewFileSet()
+ entries, err := os.ReadDir(invertedDir)
+ require.NoError(t, err)
+
+ var found *ast.FuncDecl
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ path := filepath.Join(invertedDir, name)
+ file, parseErr := parser.ParseFile(fileSet, path, nil, parser.AllErrors)
+ require.NoError(t, parseErr, "parse %s", path)
+ for _, decl := range file.Decls {
+ funcDecl, ok := decl.(*ast.FuncDecl)
+ if ok && funcDecl.Name.Name == "BuildQuery" && funcDecl.Recv != nil {
+ found = funcDecl
+ }
+ }
+ }
+ require.NotNil(t, found, "pkg/index/inverted's *store must still declare a BuildQuery method (index.SeriesStore requires it)")
+ // A stub this size is a short body: a handful of statements, no loops,
+ // no composite literals building a query tree. More than that suggests
+ // BuildQuery stopped being a stub.
+ require.LessOrEqualf(t, len(found.Body.List), 3, "pkg/index/inverted's BuildQuery has grown past a trivial always-failing stub (%d statements)", len(found.Body.List))
+}
+
+// TestRetiredSeriesSymbolsAreGone is the "actually deleted, not merely
+// unreferenced" half of the NIDX-03 §10 proof: it parses every non-test
+// pkg/index/inverted source file's top-level declarations and fails if any
+// nidx03RetiredSeriesSymbols name is still declared there.
+func TestRetiredSeriesSymbolsAreGone(t *testing.T) {
+ root := repositoryRoot(t)
+ invertedDir := filepath.Join(root, "pkg", "index", "inverted")
+ entries, err := os.ReadDir(invertedDir)
+ require.NoError(t, err)
+
+ retired := make(map[string]bool, len(nidx03RetiredSeriesSymbols))
+ for _, name := range nidx03RetiredSeriesSymbols {
+ retired[name] = true
+ }
+
+ fileSet := token.NewFileSet()
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ path := filepath.Join(invertedDir, name)
+ file, parseErr := parser.ParseFile(fileSet, path, nil, parser.AllErrors)
+ require.NoError(t, parseErr, "parse %s", path)
+ for _, decl := range file.Decls {
+ switch d := decl.(type) {
+ case *ast.FuncDecl:
+ // Checks both package-level functions and methods: none of
+ // nidx03RetiredSeriesSymbols's four names were ever declared
+ // as a method before removal, but checking methods too keeps
+ // this guard correct if one were ever reintroduced as one
+ // (for example moved onto *store the way BuildQuery was kept
+ // as a method, instead of being removed outright).
+ if retired[d.Name.Name] {
+ t.Errorf("%s still declares removed symbol %q (NIDX-03 §10)", path, d.Name.Name)
+ }
+ case *ast.GenDecl:
+ for _, spec := range d.Specs {
+ if valueSpec, ok := spec.(*ast.ValueSpec); ok {
+ for _, declaredName := range valueSpec.Names {
+ if retired[declaredName.Name] {
+ t.Errorf("%s still declares removed symbol %q (NIDX-03 §10)", path, declaredName.Name)
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+// goListDeps runs `go list -deps <target>` from root and returns the
+// dependency import paths, one per line.
+func goListDeps(t *testing.T, root, target string) []string {
+ t.Helper()
+ command := exec.Command("go", "list", "-deps", target)
+ command.Dir = root
+ output, err := command.Output()
+ require.NoError(t, err, "go list -deps %s", target)
+ return strings.Split(strings.TrimSpace(string(output)), "\n")
+}
diff --git a/banyand/internal/storage/nidx03_rollback_write_test.go b/banyand/internal/storage/nidx03_rollback_write_test.go
new file mode 100644
index 000000000..71a89104b
--- /dev/null
+++ b/banyand/internal/storage/nidx03_rollback_write_test.go
@@ -0,0 +1,514 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// NIDX-03 §12 item 2 rollback proof, write side.
+//
+// TestGenerateNIDX03RollbackCorpus is a one-off generator, gated behind
+// NIDX03_ROLLBACK_WRITE so it never runs in CI: it writes a representative
+// sidx workload with the CURRENT (native-only) code -- normal-mode and
+// index-mode series, an update (upsert of an already-live identity), an
+// external segment receive, and enough volume to cross the native owner's
+// default compaction threshold -- to NIDX03_ROLLBACK_DIR (default
+// /mnt/d/tmp-gao-build/nidx03-rollback), then runs the same fixed set of
+// queries (exact/prefix/wildcard lookup, index-order sort, projection,
+// timestamps/versions) the design's rollback test names and dumps the
+// results as JSON beside the data. docs/design/0.12.0/native-inverted-index/
+// verification/nidx-03-series-cutover/README.md records the procedure and
+// the comparison against the previous release's own reader, run from the
+// 735e9ad2 export.
+//
+// A second run, after killing (not closing) a background writer mid-batch
+// and copying the directory, produces the crash-cut variant's corpus.
+func TestGenerateNIDX03RollbackCorpus(t *testing.T) {
+ if os.Getenv("NIDX03_ROLLBACK_WRITE") == "" {
+ t.Skip("one-off rollback-proof corpus generator; set NIDX03_ROLLBACK_WRITE=1 to (re)run it")
+ }
+ root := nidx03RollbackDir(t)
+ require.NoError(t, os.RemoveAll(root))
+ require.NoError(t, os.MkdirAll(root, 0o755))
+
+ writeNIDX03NormalCorpus(t, filepath.Join(root, "normal"))
+ writeNIDX03IndexModeCorpus(t, filepath.Join(root, "indexmode"))
+ writeNIDX03CrashCutCorpus(t, filepath.Join(root, "crashcut"))
+}
+
+func nidx03RollbackDir(t *testing.T) string {
+ t.Helper()
+ if dir := os.Getenv("NIDX03_ROLLBACK_DIR"); dir != "" {
+ return dir
+ }
+ return "/mnt/d/tmp-gao-build/nidx03-rollback"
+}
+
+// nidx03RollbackQueryResult is the comparable, JSON-serializable shape both
+// the new-code writer (this file) and the previous-release reader (ported
+// into the 735e9ad2 export) dump for the same fixed set of queries. Byte
+// slices are hex-encoded by encoding/json's default []byte handling (base64,
+// actually) -- irrelevant here since both sides use the same Go encoding/json
+// package and only ever compare the resulting text for equality, never
+// decode it independently.
+//
+// cross-version diff depends on (kept identical in the 735e9ad2-ported read
+// side); it is not a hot allocation path, so pointer-byte packing does not
+// matter here.
+//
+//nolint:govet // field order is the JSON key order this proof's byte-for-byte
+type nidx03RollbackQueryResult struct {
+ Name string `json:"name"`
+ Identities []string `json:"identities"`
+ Timestamps []int64 `json:"timestamps,omitempty"`
+ Versions []int64 `json:"versions,omitempty"`
+ Projection []string `json:"projection,omitempty"`
+ SortedHex []string `json:"sortedHex,omitempty"`
+ FieldValues map[string]string `json:"fieldValues,omitempty"`
+}
+
+func writeNIDX03QueryResults(t *testing.T, path string, results []nidx03RollbackQueryResult) {
+ t.Helper()
+ data, err := json.MarshalIndent(results, "", " ")
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(path, data, 0o600))
+}
+
+const (
+ nidx03RollbackRegionKey = "region"
+ nidx03RollbackScoreRule = uint32(55)
+)
+
+func nidx03RollbackSeries(service, instance string) *pbv1.Series {
+ return &pbv1.Series{
+ Subject: "nidx03_rollback",
+ EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: service}}},
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: instance}}},
+ },
+ }
+}
+
+func nidx03RollbackDoc(t *testing.T, service, instance, region string, score, timestamp, version int64) index.Document {
+ t.Helper()
+ series := nidx03RollbackSeries(service, instance)
+ require.NoError(t, series.Marshal())
+ regionField := index.NewBytesField(index.FieldKey{TagName: nidx03RollbackRegionKey}, []byte(region))
+ regionField.Store, regionField.Index, regionField.NoSort = true, true, true
+ scoreField := index.NewBytesField(index.FieldKey{IndexRuleID: nidx03RollbackScoreRule}, fmt.Appendf(nil, "%010d", score))
+ scoreField.Store, scoreField.Index = true, true
+ return index.Document{
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Fields: []index.Field{regionField, scoreField},
+ Timestamp: timestamp,
+ Version: version,
+ }
+}
+
+// writeNIDX03NormalCorpus writes 4 services x 5 instances (20 series) across
+// 2 regions, an update that changes one series' region and score, enough
+// additional batches to cross the owner's default compaction threshold (16
+// segments), and an externally-received segment built by a second, separate
+// series index -- then captures exact/prefix/wildcard/sorted query results.
+func writeNIDX03NormalCorpus(t *testing.T, dir string) {
+ t.Helper()
+ ctx := context.Background()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ services := []string{"svcA", "svcB", "svcC", "svcD"}
+ var allDocs index.Documents
+ n := int64(0)
+ for _, svc := range services {
+ for i := 0; i < 5; i++ {
+ instance := fmt.Sprintf("inst-%02d", i)
+ region := "us"
+ if i%2 == 0 {
+ region = "eu"
+ }
+ n++
+ allDocs = append(allDocs, nidx03RollbackDoc(t, svc, instance, region, n*10, 1000+n, 1))
+ }
+ }
+ require.NoError(t, si.Insert(allDocs))
+
+ // Update: svcA/inst-00 moves region and gets a new score + version, same
+ // identity (Update is upsert: "last writer wins, no version check").
+ update := nidx03RollbackDoc(t, "svcA", "inst-00", "ap", 9999, 5000, 2)
+ require.NoError(t, si.Update(index.Documents{update}))
+
+ // Cross the native owner's default compaction threshold (16 segments):
+ // each Insert of brand-new identities below admits as its own segment.
+ for b := 0; b < 20; b++ {
+ filler := index.Documents{nidx03RollbackDoc(t, "filler", fmt.Sprintf("b%03d", b), "us", int64(b), int64(b+1), 1)}
+ require.NoError(t, si.Insert(filler))
+ }
+ // 20+ inserted batches cross the native owner's default compaction
+ // threshold (16 segments; pkg/index/native/owner.go's runMaintenance).
+
+ // External segment receive: a second, separate series index writes one
+ // committed segment this one's EnableExternalSegments then ingests raw.
+ externalDir := filepath.Join(dir, "..", "external-source")
+ extSI, err := newSeriesIndex(ctx, externalDir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ require.NoError(t, extSI.Insert(index.Documents{nidx03RollbackDoc(t, "svcExternal", "inst-00", "us", 42, 6000, 1)}))
+ require.NoError(t, extSI.Close())
+ segBytes := nidx03SingleSegmentFile(t, filepath.Join(externalDir, "sidx"))
+ streamer, err := si.EnableExternalSegments()
+ require.NoError(t, err)
+ require.NoError(t, streamer.StartSegment())
+ require.NoError(t, streamer.WriteChunk(segBytes))
+ require.NoError(t, streamer.CompleteSegment())
+
+ results := nidx03CaptureNormalQueries(t, si)
+ before, _ := si.Stats()
+ require.NoError(t, si.owner.Compact(ctx))
+ after, _ := si.Stats()
+ require.Equal(t, before, after, "compaction must not change the live document count")
+ // Re-run the same queries post-compaction: merging must not change any
+ // result.
+ require.Equal(t, results, nidx03CaptureNormalQueries(t, si))
+ require.NoError(t, si.Close())
+ writeNIDX03QueryResults(t, filepath.Join(dir, "..", "normal-results.json"), results)
+}
+
+// nidx03SingleSegmentFile returns the bytes of the one *.seg file under dir.
+func nidx03SingleSegmentFile(t *testing.T, dir string) []byte {
+ t.Helper()
+ matches, err := filepath.Glob(filepath.Join(dir, "*.seg"))
+ require.NoError(t, err)
+ require.Len(t, matches, 1, "expected exactly one segment file in %s", dir)
+ data, err := os.ReadFile(matches[0])
+ require.NoError(t, err)
+ return data
+}
+
+// nidx03CaptureNormalQueries runs the fixed query set this proof compares
+// across releases and returns the results in a stable (sorted) order.
+func nidx03CaptureNormalQueries(t *testing.T, si *seriesIndex) []nidx03RollbackQueryResult {
+ t.Helper()
+ ctx := context.Background()
+ var results []nidx03RollbackQueryResult
+
+ // Exact: the updated series must show its new region/score/version, not
+ // the pre-update one.
+ exact, _, err := si.Search(ctx, []*pbv1.Series{nidx03RollbackSeries("svcA", "inst-00")}, IndexSearchOpts{
+ Projection: []index.FieldKey{{TagName: nidx03RollbackRegionKey}},
+ })
+ require.NoError(t, err)
+ results = append(results, nidx03RollbackResult(t, "exact-updated-svcA-inst-00", exact, nil))
+
+ // Exact: the externally received series.
+ ext, _, err := si.Search(ctx, []*pbv1.Series{nidx03RollbackSeries("svcExternal", "inst-00")}, IndexSearchOpts{})
+ require.NoError(t, err)
+ results = append(results, nidx03RollbackResult(t, "exact-external-svcExternal-inst-00", ext, nil))
+
+ // Prefix: every svcB instance.
+ prefixQuery := &pbv1.Series{Subject: "nidx03_rollback", EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svcB"}}}, pbv1.AnyTagValue,
+ }}
+ prefix, _, err := si.Search(ctx, []*pbv1.Series{prefixQuery}, IndexSearchOpts{})
+ require.NoError(t, err)
+ results = append(results, nidx03RollbackResult(t, "prefix-svcB", prefix, nil))
+
+ // Wildcard: inst-02 across every service.
+ wildcardQuery := &pbv1.Series{Subject: "nidx03_rollback", EntityValues: []*modelv1.TagValue{
+ pbv1.AnyTagValue, {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "inst-02"}}},
+ }}
+ wildcard, _, err := si.Search(ctx, []*pbv1.Series{wildcardQuery}, IndexSearchOpts{})
+ require.NoError(t, err)
+ results = append(results, nidx03RollbackResult(t, "wildcard-inst-02", wildcard, nil))
+
+ // Index-order sort over svcC's instances by the score field.
+ svcCQuery := &pbv1.Series{Subject: "nidx03_rollback", EntityValues: []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "svcC"}}}, pbv1.AnyTagValue,
+ }}
+ order := &index.OrderBy{Type: index.OrderByTypeIndex, Sort: modelv1.Sort_SORT_ASC, Index: &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Id: nidx03RollbackScoreRule},
+ }}
+ sorted, sortedValues, err := si.Search(ctx, []*pbv1.Series{svcCQuery}, IndexSearchOpts{Order: order, PreloadSize: 10})
+ require.NoError(t, err)
+ results = append(results, nidx03RollbackResult(t, "sorted-svcC-by-score", sorted, sortedValues))
+
+ return results
+}
+
+func nidx03RollbackResult(t *testing.T, name string, sd SeriesData, sortedValues [][]byte) nidx03RollbackQueryResult {
+ t.Helper()
+ type row struct {
+ identity string
+ sorted string
+ timestamp int64
+ version int64
+ }
+ rows := make([]row, 0, len(sd.SeriesList))
+ for i, series := range sd.SeriesList {
+ var sortedHex string
+ if i < len(sortedValues) {
+ sortedHex = fmt.Sprintf("%x", sortedValues[i])
+ }
+ rows = append(rows, row{
+ identity: nidx03SeriesText(t, series),
+ timestamp: sd.Timestamps[i],
+ version: sd.Versions[i],
+ sorted: sortedHex,
+ })
+ }
+ if len(sortedValues) == 0 {
+ // Unordered results: sort rows by identity for a deterministic
+ // comparison across two independent readers.
+ sort.Slice(rows, func(i, j int) bool { return rows[i].identity < rows[j].identity })
+ }
+ result := nidx03RollbackQueryResult{Name: name}
+ for _, r := range rows {
+ result.Identities = append(result.Identities, r.identity)
+ result.Timestamps = append(result.Timestamps, r.timestamp)
+ result.Versions = append(result.Versions, r.version)
+ if r.sorted != "" {
+ result.SortedHex = append(result.SortedHex, r.sorted)
+ }
+ }
+ return result
+}
+
+func nidx03SeriesText(t *testing.T, series *pbv1.Series) string {
+ t.Helper()
+ parts := make([]string, 0, len(series.EntityValues))
+ for _, v := range series.EntityValues {
+ parts = append(parts, v.GetStr().GetValue())
+ }
+ return fmt.Sprintf("%v", parts)
+}
+
+// writeNIDX03IndexModeCorpus writes an index-mode (SearchWithoutSeries)
+// workload: two subjects, one of which receives an update.
+func writeNIDX03IndexModeCorpus(t *testing.T, dir string) {
+ t.Helper()
+ ctx := context.Background()
+ si, err := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+
+ buildIndexModeDoc := func(subject, service string, score, timestamp, version int64) index.Document {
+ var series pbv1.Series
+ series.Subject = subject
+ series.EntityValues = []*modelv1.TagValue{{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: service}}}}
+ require.NoError(t, series.Marshal())
+ imName := index.NewStringField(index.FieldKey{TagName: index.IndexModeName}, subject)
+ imName.Index, imName.NoSort = true, true
+ scoreField := index.NewBytesField(index.FieldKey{IndexRuleID: nidx03RollbackScoreRule}, fmt.Appendf(nil, "%010d", score))
+ scoreField.Store, scoreField.Index = true, true
+ return index.Document{
+ EntityValues: append([]byte(nil), series.Buffer...),
+ Fields: []index.Field{imName, scoreField},
+ Timestamp: timestamp,
+ Version: version,
+ }
+ }
+
+ var docs index.Documents
+ for i := 0; i < 5; i++ {
+ docs = append(docs, buildIndexModeDoc("idx_measure", fmt.Sprintf("svc-%02d", i), int64(i*10), int64(2000+i), 1))
+ }
+ docs = append(docs, buildIndexModeDoc("other_measure", "svc-other", 999, 3000, 1))
+ require.NoError(t, si.Insert(docs))
+ // Update one row in place.
+ require.NoError(t, si.Update(index.Documents{buildIndexModeDoc("idx_measure", "svc-00", 12345, 4000, 2)}))
+
+ sd, _, err := si.SearchWithoutSeries(ctx, IndexSearchOpts{IndexModeSubject: "idx_measure"})
+ require.NoError(t, err)
+ results := []nidx03RollbackQueryResult{nidx03RollbackResult(t, "indexmode-idx_measure", sd, nil)}
+ require.NoError(t, si.Close())
+ writeNIDX03QueryResults(t, filepath.Join(dir, "..", "indexmode-results.json"), results)
+}
+
+// envNIDX03CrashCutSubprocess marks a `go test` process as the crash-cut
+// writer subprocess (see TestNIDX03CrashCutWriterSubprocess) rather than the
+// normal test binary entry point; envNIDX03CrashCutSubprocessDir names the
+// directory it writes to.
+const (
+ envNIDX03CrashCutSubprocess = "NIDX03_CRASHCUT_SUBPROCESS"
+ envNIDX03CrashCutSubprocessDir = "NIDX03_CRASHCUT_SUBPROCESS_DIR"
+ nidx03CrashCutReadyLine = "NIDX03_CRASHCUT_MIDCUT_WRITTEN"
+)
+
+// TestNIDX03CrashCutWriterSubprocess is not a normal test: it only does
+// anything when envNIDX03CrashCutSubprocess is set, which only
+// writeNIDX03CrashCutCorpus's exec.Command sets. It opens a series index,
+// writes a batch that becomes durable ("precut"), writes a second batch that
+// may still be in-memory-only ("midcut"), prints nidx03CrashCutReadyLine to
+// stdout once that second Insert call has returned, and then blocks forever
+// -- it never calls Close, so a real SIGKILL (not a copy of a live
+// directory) is the only way this process ends, exactly as a killed
+// production writer would.
+func TestNIDX03CrashCutWriterSubprocess(t *testing.T) {
+ dir := os.Getenv(envNIDX03CrashCutSubprocessDir)
+ if os.Getenv(envNIDX03CrashCutSubprocess) == "" || dir == "" {
+ t.Skip("crash-cut writer subprocess entry point; not a normal test")
+ }
+ ctx := context.Background()
+ // A positive flush timeout selects asynchronous persistence
+ // (PersistInterval > 0), matching production's default -- Insert
+ // returns once the batch is admitted in memory, before the background
+ // persist cycle commits it to disk.
+ si, err := newSeriesIndex(ctx, dir, 5, 0, nil, &testRootLease{})
+ require.NoError(t, err)
+ require.NoError(t, si.Insert(index.Documents{nidx03RollbackDoc(t, "precut", "inst-00", "us", 1, 100, 1)}))
+ // Wait for "precut" to become durable (not a fixed sleep racing
+ // PersistInterval): poll the owner's own durable-generation watermark,
+ // the same mechanism nidx03SettleCompaction uses, until it reaches the
+ // generation "precut"'s Insert published.
+ view, acquireErr := si.owner.Acquire(ctx)
+ require.NoError(t, acquireErr)
+ precutGeneration := view.Generation()
+ require.NoError(t, view.Close())
+ require.Eventually(t, func() bool {
+ return si.owner.DurableGeneration() >= precutGeneration
+ }, 30*time.Second, 20*time.Millisecond, "'precut' never became durable")
+ require.NoError(t, si.Insert(index.Documents{nidx03RollbackDoc(t, "midcut", "inst-00", "us", 2, 200, 1)}))
+ fmt.Println(nidx03CrashCutReadyLine)
+ select {} // wait to be SIGKILLed; never reached otherwise.
+}
+
+// writeNIDX03CrashCutCorpus spawns this same test binary as a real OS
+// subprocess running TestNIDX03CrashCutWriterSubprocess, waits for it to
+// report that its second ("midcut") batch has been admitted, and sends it
+// SIGKILL -- a real crash, not a directory copied out from under a still-open
+// owner. It then separately writes a reference copy of the same "precut"
+// write path (closed gracefully) to pin what a from-the-start-correct reader
+// shows, and explicitly asserts the crash-cut directory's own state: the
+// batch that had time to persist ("precut") is present, and the directory a
+// previous release opens is a structurally complete generation (it may or
+// may not include "midcut", depending on exactly when the kill lands -- that
+// race is the point of a crash-cut, not a bug in this harness).
+func writeNIDX03CrashCutCorpus(t *testing.T, dir string) {
+ t.Helper()
+ ctx := context.Background()
+ crashDir := filepath.Join(dir, "sidx-parent")
+ require.NoError(t, os.MkdirAll(crashDir, 0o755))
+
+ executable, err := os.Executable()
+ require.NoError(t, err)
+ cmd := exec.Command(executable, "-test.run=^TestNIDX03CrashCutWriterSubprocess$", "-test.v")
+ cmd.Env = append(os.Environ(), envNIDX03CrashCutSubprocess+"=1", envNIDX03CrashCutSubprocessDir+"="+crashDir)
+ stdout, pipeErr := cmd.StdoutPipe()
+ require.NoError(t, pipeErr)
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Start())
+
+ ready := make(chan error, 1)
+ go func() {
+ scanner := bufio.NewScanner(stdout)
+ for scanner.Scan() {
+ if strings.Contains(scanner.Text(), nidx03CrashCutReadyLine) {
+ ready <- nil
+ return
+ }
+ }
+ ready <- fmt.Errorf("crash-cut writer subprocess exited before signaling %q: %w", nidx03CrashCutReadyLine, scanner.Err())
+ }()
+ select {
+ case readyErr := <-ready:
+ require.NoError(t, readyErr)
+ case <-time.After(30 * time.Second):
+ _ = cmd.Process.Kill()
+ t.Fatal("crash-cut writer subprocess never signaled readiness")
+ }
+ // The subprocess is blocked in `select {}`, past its own last write
+ // call: SIGKILL now is a real crash mid-persist, not a race with its own
+ // shutdown path (it has none -- it never calls Close).
+ require.NoError(t, cmd.Process.Signal(syscall.SIGKILL))
+ _ = cmd.Wait() // exits non-zero (killed); only the directory it left behind matters.
+
+ // The crash-cut directory itself, exactly as the kill left it, is what
+ // writeNIDX03NormalCorpus's sibling comparison and the previous release
+ // both open: no copy, no post-processing.
+ // The subprocess's own root is crashDir, so newSeriesIndex created
+ // crashDir/sidx (path.Join(root, seriesIndexDirName)); copy THAT, not
+ // crashDir itself, into this corpus's conventional dir/sidx layout.
+ require.NoError(t, nidx03CopyDir(filepath.Join(crashDir, "sidx"), filepath.Join(dir, "sidx")))
+
+ // Open the crash-cut directory with the current code too (a second,
+ // independent read, exactly like the previous release's own read-side
+ // test performs) and assert explicitly what a real crash leaves:
+ // "precut" (which had 200ms to persist) must be present; the directory
+ // must open at all, proving it is a complete generation, not a torn one.
+ reopened, reopenErr := newSeriesIndex(ctx, dir, 0, 0, nil, &testRootLease{})
+ require.NoError(t, reopenErr, "a crash-cut directory must still open as a complete generation")
+ sd, _, searchErr := reopened.Search(ctx, []*pbv1.Series{nidx03RollbackSeries("precut", "inst-00")}, IndexSearchOpts{})
+ require.NoError(t, searchErr)
+ require.Len(t, sd.SeriesList, 1, "the pre-crash, already-persisted 'precut' write must survive the crash")
+ results := []nidx03RollbackQueryResult{nidx03RollbackResult(t, "crashcut-precut-reference", sd, nil)}
+ require.NoError(t, reopened.Close())
+ writeNIDX03QueryResults(t, filepath.Join(dir, "crashcut-results.json"), results)
+}
+
+// nidx03CopyDir recursively copies src to dst. The crash-cut writer always
+// SIGKILLed, never copied live; this moves ITS directory (now static, since
+// the process that wrote to it is dead) into this corpus's conventional
+// `dir/sidx` layout, which every other writeNIDX03*Corpus function and the
+// previous-release read side also use.
+func nidx03CopyDir(src, dst string) error {
+ return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ relative, relErr := filepath.Rel(src, path)
+ if relErr != nil {
+ return relErr
+ }
+ target := filepath.Join(dst, relative)
+ if info.IsDir() {
+ return os.MkdirAll(target, info.Mode())
+ }
+ in, openErr := os.Open(path)
+ if openErr != nil {
+ return openErr
+ }
+ defer func() { _ = in.Close() }()
+ out, createErr := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
+ if createErr != nil {
+ return createErr
+ }
+ _, copyErr := io.Copy(out, in)
+ closeErr := out.Close()
+ if copyErr != nil {
+ return copyErr
+ }
+ return closeErr
+ })
+}
diff --git a/banyand/internal/storage/rotation_test.go b/banyand/internal/storage/rotation_test.go
index 47e1c46c2..436c56b27 100644
--- a/banyand/internal/storage/rotation_test.go
+++ b/banyand/internal/storage/rotation_test.go
@@ -161,6 +161,7 @@ func setUpDB(t *testing.T, ttlDays ...int) (*database[*MockTSTable, any], timest
}
TSDBOpts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: ttl},
@@ -240,6 +241,7 @@ func TestRotationDisabled(t *testing.T) {
defer defFn()
TSDBOpts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 3},
TTL: IntervalRule{Unit: DAY, Num: 30},
@@ -317,6 +319,7 @@ func TestTickRejectsZeroTimestamp(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 00dd5b67d..7f53bbefe 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -26,6 +26,7 @@ import (
"path/filepath"
"sort"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"time"
@@ -35,7 +36,8 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
banyanfs "github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ idxmetrics "github.com/apache/skywalking-banyandb/pkg/index/metrics"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -72,7 +74,7 @@ type segment[T TSTable, O any] struct {
index *seriesIndex
sLst atomic.Pointer[[]*shard[T]]
*segmentCache
- indexMetrics *inverted.Metrics
+ indexMetrics *idxmetrics.Metrics
lfs banyanfs.FileSystem
position common.Position
timestamp.TimeRange
@@ -83,6 +85,12 @@ type segment[T TSTable, O any] struct {
refCount int32
mustBeDeleted uint32
id segmentID
+ // closeFailed records that the most recent closeResourcesLocked call
+ // could not cleanly close the series index (the owner's final
+ // persistence flush reported an error). The segment itself stays usable
+ // -- its directory and shards are untouched -- but the failure is
+ // surfaced here instead of panicking the node, per NIDX-03 §8/§11.
+ closeFailed atomic.Bool
}
func (sc *segmentController[T, O]) openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix string, groupCache *groupCache,
@@ -210,7 +218,8 @@ func (s *segment[T, O]) initialize(ctx context.Context) error {
return s.position
})
- sir, err := newSeriesIndex(ctx, s.location, s.tsdbOpts.SeriesIndexFlushTimeoutSeconds, s.tsdbOpts.SeriesIndexCacheMaxBytes, s.indexMetrics)
+ sir, err := newSeriesIndex(ctx, s.location, s.tsdbOpts.SeriesIndexFlushTimeoutSeconds, s.tsdbOpts.SeriesIndexCacheMaxBytes,
+ s.indexMetrics, s.tsdbOpts.RootLease)
if err != nil {
return errors.Wrap(errOpenDatabase, errors.WithMessage(err, "create series index controller failed").Error())
}
@@ -223,6 +232,18 @@ func (s *segment[T, O]) initialize(ctx context.Context) error {
return errors.Wrap(errOpenDatabase, errors.WithMessage(err, "load shards failed").Error())
}
+ // A successful reopen clears a previous close's failure: closeFailed
+ // records only the most recent closeResourcesLocked call (see its
+ // field doc), and this segment's directory and on-disk data are exactly
+ // what loadShards and the series index above just validated by opening
+ // them. Leaving the stale flag set would make a segment that reopens
+ // and operates normally forever report CloseFailed()==true from one
+ // transient persistence fault, which is strictly worse than losing the
+ // signal: nothing in this codebase refuses to reuse a failed segment
+ // today, so the flag's only value is observability, and a permanently
+ // stuck true defeats that.
+ s.closeFailed.Store(false)
+
s.l.Info().Stringer("seg", s).Msg("segment initialized")
return nil
}
@@ -261,7 +282,7 @@ func (s *segment[T, O]) resetIndex() {
s.mu.RLock()
defer s.mu.RUnlock()
if s.index != nil {
- s.index.store.Reset()
+ s.index.ResetCache()
}
}
@@ -280,7 +301,11 @@ func (s *segment[T, O]) collectOpenMetrics(shardMetrics Metrics) bool {
sh.table.Collect(shardMetrics)
}
}
- s.index.store.CollectMetrics(s.index.p.SegLabelValues()...)
+ // The native series index does not yet publish the detailed
+ // idxmetrics.Metrics gauges (merge counts, analysis time, cache hit/miss,
+ // ...) the retired bluge store reported; pkg/index/native.Owner exposes
+ // no equivalent introspection today. Stats() (dataCount/dataSizeBytes)
+ // still flows through SeriesIndexStats below.
return true
}
@@ -308,10 +333,17 @@ func (s *segment[T, O]) DecRef() {
// closeResourcesLocked closes the segment's series index and shards but keeps
// its on-disk directory, leaving it closed (index==nil) and reopenable. Must be
// called with s.mu held. Idempotent.
+//
+// A series-index close error (for example the native owner's final
+// persistence flush discovering a durability fault) is logged and recorded
+// on closeFailed instead of panicking the node: the in-memory root was
+// already visible and the on-disk directory is untouched by the close
+// itself, so the segment stays usable and reopenable (NIDX-03 §8/§11).
func (s *segment[T, O]) closeResourcesLocked() {
if s.index != nil {
if err := s.index.Close(); err != nil {
- s.l.Panic().Err(err).Msg("failed to close the series index")
+ s.l.Error().Err(err).Stringer("seg", s).Msg("failed to close the series index; marking the segment failed")
+ s.closeFailed.Store(true)
}
s.index = nil
}
@@ -368,10 +400,16 @@ func (s *segment[T, O]) Location() string {
return s.location
}
+// CloseFailed reports whether the most recent closeResourcesLocked call
+// could not cleanly close this segment's series index. See closeFailed.
+func (s *segment[T, O]) CloseFailed() bool {
+ return s.closeFailed.Load()
+}
+
// SeriesIndexStats returns the series index document count and on-disk size.
// An open segment is reported from its live index; a closed segment is read
-// from disk read-only (via OpenReader + directory walk), so inspecting a cold
-// segment never reopens its writable index.
+// from disk read-only (via a native read-only generation), so inspecting a
+// cold segment never reopens its writable index.
func (s *segment[T, O]) SeriesIndexStats() (int64, int64) {
s.mu.RLock()
if s.index != nil {
@@ -385,7 +423,7 @@ func (s *segment[T, O]) SeriesIndexStats() (int64, int64) {
// read-only. Best-effort -- a missing/unflushed/concurrently-removed index
// reports 0.
indexPath := filepath.Join(s.location, seriesIndexDirName)
- count, err := inverted.ReadOnlyDocCount(indexPath)
+ count, err := closedSeriesIndexDocCount(indexPath)
if err != nil {
s.l.Debug().Err(err).Str("path", indexPath).Msg("closed series index has no readable doc count")
}
@@ -393,6 +431,25 @@ func (s *segment[T, O]) SeriesIndexStats() (int64, int64) {
return count, int64(size)
}
+// closedSeriesIndexDocCount opens indexPath's newest committed generation
+// read-only (no lock, no reopen of a writable owner) and reports its live
+// document count. A directory with no committed generation yet reports 0,
+// not an error. The open is strict (native.OpenReadOnlyGenerationStrict, the
+// same fail-closed mode a writer reopening the directory uses): a corrupt
+// newest manifest reports an error rather than silently falling back to an
+// older generation's count, which would misreport a closed segment's size.
+func closedSeriesIndexDocCount(indexPath string) (int64, error) {
+ generation, err := native.OpenReadOnlyGenerationStrict(indexPath)
+ if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return 0, nil
+ }
+ return 0, err
+ }
+ defer func() { _ = generation.Close() }()
+ return generation.VisibleDocCount()
+}
+
// snapshotInto writes a point-in-time snapshot of this segment under dst.
//
// It NEVER reopens a closed segment -- reopening an idle-closed cold segment is
@@ -424,26 +481,95 @@ func (s *segment[T, O]) snapshotInto(dst string) (bool, error) {
return s.snapshotClosed(dst)
}
-// snapshotClosed hard-links the whole quiescent segment directory into dst.
-// Must be called with s.mu held and s.index == nil.
+// snapshotClosed hard-links a quiescent segment directory into dst. Every
+// shard directory is hard-linked whole (includeInClosedSnapshot's existing
+// exclusions); the series index directory is narrowed further to only its
+// newest committed generation's manifest and the segment files it
+// references (NIDX-03 §8), so a cold sidx directory that still holds
+// superseded snapshots or segments the owner has not yet garbage collected
+// does not carry them into the backup. Must be called with s.mu held and
+// s.index == nil.
func (s *segment[T, O]) snapshotClosed(dst string) (bool, error) {
segDir := filepath.Base(s.location)
segPath := filepath.Join(dst, segDir)
- if err := s.lfs.CreateHardLink(s.location, segPath, includeInClosedSnapshot); err != nil {
+ sidxPath := filepath.Join(s.location, seriesIndexDirName)
+ keep, err := newestSeriesIndexGenerationFiles(sidxPath)
+ if err != nil {
+ return false, errors.Wrapf(err, "failed to resolve newest series index generation for segment %s", segDir)
+ }
+ if err := s.lfs.CreateHardLink(s.location, segPath, closedSnapshotFilter(sidxPath, keep)); err != nil {
return false, errors.Wrapf(err, "failed to hard-link closed segment %s", segDir)
}
+ // A series index with no committed generation yet (no admission since
+ // the segment was created) has no sidx directory on disk at all, so the
+ // walk above never visits or creates one in the destination. Create an
+ // empty one so a closed, never-written segment's snapshot has the same
+ // shape as the open path's (snapshotOpen always creates the series
+ // index directory, even for a brand-new empty index).
+ s.lfs.MkdirIfNotExist(filepath.Join(segPath, seriesIndexDirName), DirPerm)
return true, nil
}
+// newestSeriesIndexGenerationFiles returns the file names (relative to
+// sidxPath, matching native.ReadOnlyGeneration.ReferencedFiles) the series
+// index's newest committed generation references. A directory with no
+// committed generation yet (a brand-new, never-flushed series index) reports
+// an empty set, not an error. The open is strict
+// (native.OpenReadOnlyGenerationStrict): a corrupt newest manifest fails the
+// backup instead of silently hard-linking an older generation's files while
+// claiming it is current, which would make snapshotClosed drop acknowledged
+// data without reporting an error.
+func newestSeriesIndexGenerationFiles(sidxPath string) (map[string]struct{}, error) {
+ generation, err := native.OpenReadOnlyGenerationStrict(sidxPath)
+ if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return map[string]struct{}{}, nil
+ }
+ return nil, err
+ }
+ defer func() { _ = generation.Close() }()
+ files := generation.ReferencedFiles()
+ keep := make(map[string]struct{}, len(files))
+ for _, f := range files {
+ keep[f] = struct{}{}
+ }
+ return keep, nil
+}
+
+// closedSnapshotFilter wraps includeInClosedSnapshot so that, within the
+// series index directory specifically, only a file named in keep (the
+// newest committed generation's manifest and segments) is hard-linked.
+// Everything outside sidxPath keeps includeInClosedSnapshot's existing
+// behavior unchanged.
+func closedSnapshotFilter(sidxPath string, keep map[string]struct{}) func(string) bool {
+ return func(p string) bool {
+ if !includeInClosedSnapshot(p) {
+ return false
+ }
+ rel, relErr := filepath.Rel(sidxPath, p)
+ if relErr != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+ // p is sidxPath itself, or outside it entirely: unchanged behavior.
+ return true
+ }
+ _, ok := keep[rel]
+ return ok
+ }
+}
+
// includeInClosedSnapshot reports whether a file or directory under a closed
// segment should be hard-linked into a snapshot. It excludes the transient and
// non-current artifacts that the open-path snapshot never copies: the bluge
-// lock file, the failed-parts directory, the external-segment temp directory,
-// and partial ".tmp" atomic-write files. Current part directories and their
+// lock file, a seg-*/lock exclusive-lock file (lockFilename; the offline
+// index-mode copy tool's targetIdxStore creates one beside the sidx
+// directory of every target segment it writes and only removes it in
+// closeAll, so a tool crash leaves it on disk -- it must never leak into a
+// backup, the same way the TSDB root's own same-named lock file never does),
+// the failed-parts directory, the external-segment temp directory, and
+// partial ".tmp" atomic-write files. Current part directories and their
// ".snp" manifests are kept.
func includeInClosedSnapshot(p string) bool {
switch base := filepath.Base(p); {
- case base == inverted.LockFilename, base == FailedPartsDirName, base == inverted.ExternalSegmentTempDirName:
+ case base == legacyLockFilename, base == lockFilename, base == FailedPartsDirName, base == legacyExternalSegmentTempDirName:
return false
default:
return filepath.Ext(base) != ".tmp"
@@ -466,7 +592,7 @@ func (s *segment[T, O]) snapshotOpen(dst string, idx *seriesIndex) (bool, error)
indexPath := filepath.Join(segPath, seriesIndexDirName)
s.lfs.MkdirIfNotExist(indexPath, DirPerm)
- if err := idx.store.TakeFileSnapshot(indexPath); err != nil {
+ if err := idx.TakeFileSnapshot(indexPath); err != nil {
return false, errors.Wrapf(err, "failed to snapshot index for segment %s", segDir)
}
@@ -542,7 +668,7 @@ type segmentController[T TSTable, O any] struct {
metrics Metrics
opts *TSDBOpts[T, O]
l *logger.Logger
- indexMetrics *inverted.Metrics
+ indexMetrics *idxmetrics.Metrics
*groupCache
lfs banyanfs.FileSystem
position common.Position
@@ -556,11 +682,23 @@ type segmentController[T TSTable, O any] struct {
}
func newSegmentController[T TSTable, O any](ctx context.Context, location string,
- l *logger.Logger, opts TSDBOpts[T, O], indexMetrics *inverted.Metrics, metrics Metrics,
+ l *logger.Logger, opts TSDBOpts[T, O], indexMetrics *idxmetrics.Metrics, metrics Metrics,
idleTimeout time.Duration, lfs banyanfs.FileSystem, cache Cache, group string,
) *segmentController[T, O] {
clock, _ := timestamp.GetClock(ctx)
p := common.GetPosition(ctx)
+ // opts.RootLease is intentionally left nil here when the caller did not
+ // supply one: every series index now opens a pkg/index/native.Owner,
+ // which requires a non-nil lease unconditionally (NewOwner's "ownership
+ // is an executable invariant") and returns native.ErrLeaseUnavailable on
+ // a nil one. Silently substituting a no-op lease would hide a caller
+ // that forgot to configure OpenTSDB's RootLeaseFactory -- every product
+ // today (see banyand/{measure,stream,trace}/metadata.go) sets one before
+ // any segment actually opens (segments open lazily, well after OpenTSDB
+ // returns), so this never weakens a production path. Tests that
+ // construct a segmentController directly must supply their own
+ // RootLease (for example &testRootLease{}), matching the low-level
+ // series-index tests in this package.
return &segmentController[T, O]{
location: location,
opts: &opts,
diff --git a/banyand/internal/storage/segment_close_resources_test.go b/banyand/internal/storage/segment_close_resources_test.go
new file mode 100644
index 000000000..c124e32b1
--- /dev/null
+++ b/banyand/internal/storage/segment_close_resources_test.go
@@ -0,0 +1,105 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "testing"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// TestCloseResourcesLocked_SeriesIndexCloseErrorDoesNotPanic pins the NIDX-03
+// §8/§11 fix: a series-index close error (here, the native owner's
+// background persistence hitting an induced, always-failing PersistFunc test
+// seam) must be logged and recorded on the segment instead of panicking the
+// node. The segment's resources are still released (index set to nil) so it
+// remains reopenable.
+func TestCloseResourcesLocked_SeriesIndexCloseErrorDoesNotPanic(t *testing.T) {
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "error"}))
+
+ owner, err := native.NewOwner(native.OwnerOptions{
+ Lease: &testRootLease{},
+ Persist: func(context.Context, *native.ReadView) error {
+ return errors.New("induced persistence failure")
+ },
+ })
+ require.NoError(t, err)
+
+ // Admit a document with no PersistentCallback: drainPersistence (run by
+ // either the background worker or Close's own final flush) records the
+ // induced failure into o.persistErr regardless of which one gets to it
+ // first, so the eventual Owner.Close() error is deterministic.
+ require.NoError(t, owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{Identifier: []byte("series-a")}},
+ }))
+
+ si := &seriesIndex{owner: owner, l: logger.GetLogger("test")}
+ seg := &segment[*MockTSTable, any]{index: si, l: logger.GetLogger("test")}
+
+ require.NotPanics(t, func() {
+ seg.mu.Lock()
+ seg.closeResourcesLocked()
+ seg.mu.Unlock()
+ })
+ require.True(t, seg.CloseFailed(), "a series-index close error must mark the segment failed instead of panicking")
+ require.Nil(t, seg.index, "resources are still released despite the close error")
+
+ // Idempotent: a second call (as closeIfIdle/performDelete may perform)
+ // must not panic either.
+ require.NotPanics(t, func() {
+ seg.mu.Lock()
+ seg.closeResourcesLocked()
+ seg.mu.Unlock()
+ })
+}
+
+// TestSegment_Initialize_ClearsCloseFailedOnSuccessfulReopen is the L1
+// CloseFailed-lifecycle decision: closeFailed records only the most recent
+// closeResourcesLocked call, and this segment's directory and on-disk data
+// are exactly what loadShards and the series index just validated by
+// opening them again. A successful reopen must therefore clear a stale
+// closeFailed from an earlier close, rather than leaving a segment that
+// reopens and operates normally forever reporting CloseFailed()==true from
+// one transient persistence fault.
+func TestSegment_Initialize_ClearsCloseFailedOnSuccessfulReopen(t *testing.T) {
+ require.NoError(t, logger.Init(logger.Logging{Env: "dev", Level: "error"}))
+ dir, fn := setUp(require.New(t))
+ defer fn()
+
+ seg := &segment[*MockTSTable, any]{
+ location: dir,
+ l: logger.GetLogger("test"),
+ tsdbOpts: &TSDBOpts[*MockTSTable, any]{RootLease: &testRootLease{}},
+ }
+ seg.closeFailed.Store(true) // simulate a previous close failure.
+
+ seg.mu.Lock()
+ initErr := seg.initialize(context.Background())
+ seg.mu.Unlock()
+ require.NoError(t, initErr)
+ require.False(t, seg.CloseFailed(), "a successful reopen must clear a stale closeFailed from an earlier close")
+
+ seg.mu.Lock()
+ seg.closeResourcesLocked()
+ seg.mu.Unlock()
+}
diff --git a/banyand/internal/storage/segment_idle_reclaim_test.go b/banyand/internal/storage/segment_idle_reclaim_test.go
index 10d95e869..1dc0973d4 100644
--- a/banyand/internal/storage/segment_idle_reclaim_test.go
+++ b/banyand/internal/storage/segment_idle_reclaim_test.go
@@ -64,6 +64,7 @@ func newReclaimTestController(
return common.Position{Database: "test-db", Stage: "test-stage"}
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
diff --git a/banyand/internal/storage/segment_snapshot_closed_native_test.go b/banyand/internal/storage/segment_snapshot_closed_native_test.go
new file mode 100644
index 000000000..edb1636b6
--- /dev/null
+++ b/banyand/internal/storage/segment_snapshot_closed_native_test.go
@@ -0,0 +1,237 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+)
+
+// buildTestSeriesDocsNamed is buildTestSeriesDocs with a caller-chosen
+// subject, so two calls produce documents with disjoint identifiers (the
+// series ID folds the subject in) instead of InsertIfAbsent silently
+// skipping a repeat of the same entity_0..entity_{n-1} identifiers.
+func buildTestSeriesDocsNamed(t *testing.T, subject string, n int) index.Documents {
+ t.Helper()
+ var docs index.Documents
+ for i := 0; i < n; i++ {
+ var series pbv1.Series
+ series.Subject = subject
+ series.EntityValues = []*modelv1.TagValue{
+ {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: fmt.Sprintf("entity_%d", i)}}},
+ }
+ require.NoError(t, series.Marshal())
+ docs = append(docs, index.Document{EntityValues: append([]byte(nil), series.Buffer...)})
+ }
+ return docs
+}
+
+// admitSynchronously admits one document into owner and blocks for its
+// durability callback, so the caller can rely on the generation being fully
+// persisted (a new manifest and segment file on disk) before proceeding.
+func admitSynchronously(t *testing.T, owner *native.Owner, identifier string) {
+ t.Helper()
+ done := make(chan error, 1)
+ require.NoError(t, owner.Batch(context.Background(), native.Batch{
+ Documents: []native.Document{{Identifier: []byte(identifier)}},
+ PersistentCallback: func(err error) { done <- err },
+ }))
+ require.NoError(t, <-done)
+}
+
+// TestNewestSeriesIndexGenerationFilesExcludesSuperseded pins the NIDX-03 §8
+// fix directly against the two helper functions snapshotClosed uses: a sidx
+// directory holding two generations (as a live owner does between an
+// admission and the next garbage collection, or across any admission the
+// maintenance compaction threshold has not yet reached) must resolve a keep
+// set naming only the newest manifest, and closedSnapshotFilter must apply
+// that set only inside the series index directory, leaving everything
+// outside it untouched.
+func TestNewestSeriesIndexGenerationFilesExcludesSuperseded(t *testing.T) {
+ dir := t.TempDir()
+ indexPath := filepath.Join(dir, "sidx")
+ owner, err := native.NewOwner(native.OwnerOptions{Lease: &testRootLease{}, Path: indexPath})
+ require.NoError(t, err)
+ defer func() { require.NoError(t, owner.Close()) }()
+
+ // Two independent admissions each durably persist their own generation
+ // (manifest + segment). Both are far below the default compaction
+ // threshold (16 segments), so no compaction or garbage collection runs
+ // in between: generation 1's manifest is still on disk, superseded by
+ // generation 2's, when the keep set below is resolved.
+ admitSynchronously(t, owner, "a")
+ admitSynchronously(t, owner, "b")
+
+ entries, err := os.ReadDir(indexPath)
+ require.NoError(t, err)
+ var manifests []string
+ for _, e := range entries {
+ if !e.IsDir() && filepath.Ext(e.Name()) == ".snp" {
+ manifests = append(manifests, e.Name())
+ }
+ }
+ require.GreaterOrEqual(t, len(manifests), 2, "precondition: both generations' manifests are still on disk (no GC has run)")
+ sort.Strings(manifests) // "%012x.snp" zero-padded hex sorts lexicographically == numerically.
+ oldest, newest := manifests[0], manifests[len(manifests)-1]
+
+ keep, err := newestSeriesIndexGenerationFiles(indexPath)
+ require.NoError(t, err)
+ _, newestKept := keep[newest]
+ require.True(t, newestKept, "the newest manifest must be in the keep set")
+ _, oldestKept := keep[oldest]
+ require.False(t, oldestKept, "a superseded manifest must not be in the keep set")
+
+ filter := closedSnapshotFilter(indexPath, keep)
+ require.True(t, filter(filepath.Join(indexPath, newest)), "the newest manifest must be kept")
+ require.False(t, filter(filepath.Join(indexPath, oldest)), "a superseded manifest must be excluded")
+ require.True(t, filter(indexPath), "the series index directory itself must always be included so the walk descends")
+
+ // Outside indexPath entirely: unchanged includeInClosedSnapshot behavior.
+ outside := filepath.Dir(indexPath)
+ require.True(t, filter(filepath.Join(outside, "shard-0", "metadata.json")))
+ require.False(t, filter(filepath.Join(outside, "shard-0", legacyLockFilename)))
+ // L8 regression: a seg-*/lock exclusive-lock file (lockFilename) is the
+ // offline index-mode copy tool's targetIdxStore lease lock, created
+ // beside the target sidx directory and removed only on a clean
+ // closeAll. A tool crash leaves it on disk; it must never be hard-linked
+ // into a closed-segment snapshot the same way the TSDB root's own
+ // same-named lock file never is.
+ require.False(t, filter(filepath.Join(outside, lockFilename)), "a stray seg-*/lock file must be excluded from closed-segment snapshots")
+}
+
+// TestNewestSeriesIndexGenerationFilesNoCommittedGeneration verifies a sidx
+// directory with no committed generation yet (brand new, never flushed, or
+// simply absent on disk) resolves an empty keep set rather than an error.
+func TestNewestSeriesIndexGenerationFilesNoCommittedGeneration(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "sidx")
+
+ keep, err := newestSeriesIndexGenerationFiles(dir)
+ require.NoError(t, err)
+ require.Empty(t, keep)
+}
+
+// TestClosedSegmentHelpers_CorruptNewestManifestFailsClosed is the L2
+// regression. newestSeriesIndexGenerationFiles and closedSeriesIndexDocCount
+// used native.OpenReadOnlyGeneration, which falls back to the last complete
+// generation when the newest manifest is corrupt -- correct for the
+// best-effort offline tools it also serves, but wrong here: silently
+// hard-linking (or reporting the doc count of) an OLDER generation while
+// claiming it is current would make a closed-segment backup drop
+// acknowledged data with no error. Both helpers must instead open strict
+// (native.OpenReadOnlyGenerationStrict, the same fail-closed mode a writer
+// reopening the directory uses) and fail the same way native.NewOwner
+// already does on the same corrupt directory, rather than disagreeing with
+// it.
+func TestClosedSegmentHelpers_CorruptNewestManifestFailsClosed(t *testing.T) {
+ dir := t.TempDir()
+ indexPath := filepath.Join(dir, "sidx")
+ owner, err := native.NewOwner(native.OwnerOptions{Lease: &testRootLease{}, Path: indexPath})
+ require.NoError(t, err)
+ admitSynchronously(t, owner, "a")
+ admitSynchronously(t, owner, "b")
+ require.NoError(t, owner.Close())
+
+ entries, err := os.ReadDir(indexPath)
+ require.NoError(t, err)
+ var manifests []string
+ for _, e := range entries {
+ if !e.IsDir() && filepath.Ext(e.Name()) == ".snp" {
+ manifests = append(manifests, e.Name())
+ }
+ }
+ require.GreaterOrEqual(t, len(manifests), 2, "precondition: two generations' manifests are both still on disk")
+ sort.Strings(manifests)
+ newest := manifests[len(manifests)-1]
+
+ // Overwrite the newest manifest with garbage, corrupting it while
+ // leaving the older, complete generation intact on disk -- exactly the
+ // condition the lenient OpenReadOnlyGeneration would silently roll back
+ // from.
+ require.NoError(t, os.WriteFile(filepath.Join(indexPath, newest), []byte("not a valid manifest"), 0o600))
+
+ _, countErr := closedSeriesIndexDocCount(indexPath)
+ require.Error(t, countErr, "a corrupt newest manifest must fail closedSeriesIndexDocCount, not silently report an older generation's count")
+
+ _, filesErr := newestSeriesIndexGenerationFiles(indexPath)
+ require.Error(t, filesErr, "a corrupt newest manifest must fail newestSeriesIndexGenerationFiles, not silently keep an older generation's files")
+
+ // Consistency check (the coordinator's own probe): a live owner
+ // reopening the same corrupt directory must fail too, not disagree with
+ // the closed-segment helpers above.
+ _, reopenErr := native.NewOwner(native.OwnerOptions{Lease: &testRootLease{}, Path: indexPath})
+ require.Error(t, reopenErr, "NewOwner must also refuse to reopen a directory with a corrupt newest manifest")
+}
+
+// TestSnapshotClosed_ExcludesSupersededManifest drives the fix through the
+// real segment/TSDB stack: two independent synchronous admissions each
+// durably persist their own generation (manifest + segment) with no
+// compaction between them (far below the default compaction threshold), so
+// the live sidx directory still holds the first, now-superseded manifest
+// when the segment is idle-closed and snapshotted. The hard-linked copy must
+// carry only the newest manifest.
+func TestSnapshotClosed_ExcludesSupersededManifest(t *testing.T) {
+ dir := snapshotTestDir(t)
+ tsdb, sc, seg := openSnapshotTestTSDB(t, dir)
+ defer func() { require.NoError(t, tsdb.Close()) }()
+
+ require.NoError(t, seg.IndexDB().Insert(buildTestSeriesDocsNamed(t, "gen1", 2)))
+ require.NoError(t, seg.IndexDB().Insert(buildTestSeriesDocsNamed(t, "gen2", 2)))
+
+ sidxPath := filepath.Join(seg.location, seriesIndexDirName)
+ srcManifests := countExt(t, sidxPath, ".snp")
+ require.GreaterOrEqual(t, srcManifests, 2, "precondition: more than one generation's manifest is still on disk")
+
+ seg.lastAccessed.Store(time.Now().Add(-2 * time.Hour).UnixNano())
+ require.Equal(t, 1, sc.closeIdleSegments())
+ require.Nil(t, seg.index)
+
+ snapshotDir := filepath.Join(dir, "snapshot")
+ created, err := tsdb.TakeFileSnapshot(snapshotDir)
+ require.NoError(t, err)
+ require.True(t, created)
+
+ dstSidx := filepath.Join(snapshotDir, filepath.Base(seg.location), seriesIndexDirName)
+ require.Equal(t, 1, countExt(t, dstSidx, ".snp"), "only the newest committed manifest must be hard-linked")
+ require.Equal(t, int64(4), snapshotSeriesDocCount(t, snapshotDir, seg), "the snapshot copy still reads back every live document")
+}
+
+// countExt counts the regular files under dir whose extension is ext.
+func countExt(t *testing.T, dir, ext string) int {
+ t.Helper()
+ entries, err := os.ReadDir(dir)
+ require.NoError(t, err)
+ count := 0
+ for _, e := range entries {
+ if !e.IsDir() && filepath.Ext(e.Name()) == ext {
+ count++
+ }
+ }
+ return count
+}
diff --git a/banyand/internal/storage/segment_test.go b/banyand/internal/storage/segment_test.go
index 41edf7042..fb4db7df5 100644
--- a/banyand/internal/storage/segment_test.go
+++ b/banyand/internal/storage/segment_test.go
@@ -88,6 +88,7 @@ func TestSegmentOpenAndReopen(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -187,6 +188,7 @@ func TestSegmentCloseIfIdle(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -276,6 +278,7 @@ func TestCloseIdleAndSelectSegments(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -406,6 +409,7 @@ func TestOpenExistingSegmentWithShards(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, location string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -523,6 +527,7 @@ func TestDeleteExpiredSegmentsWithClosedSegments(t *testing.T) {
// Use a short TTL for testing - 3 days
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -672,6 +677,7 @@ func TestCreateSegmentWritesJSONMetadata(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -739,6 +745,7 @@ func TestOpenReadsPersistedEndTime(t *testing.T) {
group := "test-group"
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -820,6 +827,7 @@ func TestOpenFallbackOldFormatMetadata(t *testing.T) {
group := "test-group"
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -894,6 +902,7 @@ func TestSegment_IndexDB_ReturnsUntypedNilWhenClosed(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -987,6 +996,7 @@ func TestSegment_ConcurrentReopen_RefCountConsistent(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -1102,6 +1112,7 @@ func TestSegment_ConcurrentReopenAndClose_NoPanic(t *testing.T) {
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
@@ -1229,6 +1240,7 @@ func newAlignmentTestController(
})
opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ RootLease: staticTestLease{},
TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
_ timestamp.TimeRange, _ mockTSTableOpener, _ any,
) (mockTSTable, error) {
diff --git a/banyand/internal/storage/snapshot_closed_segment_test.go b/banyand/internal/storage/snapshot_closed_segment_test.go
index 5c4719dad..21c55a8e9 100644
--- a/banyand/internal/storage/snapshot_closed_segment_test.go
+++ b/banyand/internal/storage/snapshot_closed_segment_test.go
@@ -33,6 +33,7 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -81,6 +82,7 @@ func snapshotTestDir(t *testing.T) string {
func openSnapshotTSDB(t *testing.T, dir string, ttlDays int) (TSDB[*MockTSTable, any], *segmentController[*MockTSTable, any]) {
t.Helper()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: ttlDays},
@@ -329,7 +331,7 @@ func createSegmentWithSeries(t *testing.T, tsdb TSDB[*MockTSTable, any], ts time
// under snapshotDir, read-only.
func snapshotSeriesDocCount(t *testing.T, snapshotDir string, seg *segment[*MockTSTable, any]) int64 {
t.Helper()
- count, err := inverted.ReadOnlyDocCount(filepath.Join(snapshotDir, filepath.Base(seg.location), seriesIndexDirName))
+ count, err := native.ReadOnlyDocCount(filepath.Join(snapshotDir, filepath.Base(seg.location), seriesIndexDirName))
require.NoError(t, err)
return count
}
diff --git a/banyand/internal/storage/storage.go b/banyand/internal/storage/storage.go
index f9a0fdcf0..81c7f6f5f 100644
--- a/banyand/internal/storage/storage.go
+++ b/banyand/internal/storage/storage.go
@@ -32,8 +32,10 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native/criteria"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
@@ -75,12 +77,28 @@ var (
type SupplyTSDB[T TSTable] func() T
// IndexSearchOpts is the options for searching index.
+//
+//nolint:govet // option fields are grouped by purpose, not padding.
type IndexSearchOpts struct {
- Query index.Query
- Order *index.OrderBy
- TimeRange *timestamp.TimeRange
- Projection []index.FieldKey
- PreloadSize int
+ // Criteria is the filter the series index evaluates over the universe
+ // (the series matchers for Search, or IndexModeSubject for
+ // SearchWithoutSeries) via a criteria.FieldResolver-backed
+ // pkg/index/native/criteria.Filter call. A nil Criteria matches every
+ // document the universe already selected.
+ Criteria *modelv1.Criteria
+ // Fields resolves a Criteria condition's tag name to the engine field
+ // (and, for MATCH, the analyzer) that carries it. Required whenever
+ // Criteria is non-nil.
+ Fields criteria.FieldResolver
+ // IndexModeSubject is the literal `_im_name` term value SearchWithoutSeries
+ // scopes its universe to -- the index-mode measure's name. It is ignored
+ // by Search, which scopes its universe to the supplied series matchers
+ // instead.
+ IndexModeSubject string
+ Order *index.OrderBy
+ TimeRange *timestamp.TimeRange
+ Projection []index.FieldKey
+ PreloadSize int
}
// FieldResult is the result of a field.
@@ -89,12 +107,21 @@ type FieldResult map[string][]byte
// FieldResultList is a list of FieldResult.
type FieldResultList []FieldResult
-// SeriesData is the result of a series.
+// SeriesData is the result of a series. Timestamps, Versions, TimestampSet
+// and VersionSet are always exactly len(SeriesList) long -- one entry per
+// row, aligned 1:1 with SeriesList by position. TimestampSet[i]/VersionSet[i]
+// report whether Timestamps[i]/Versions[i] holds a real value (true) or the
+// zero value because that row carried none (false). This replaces a legacy
+// convention that appended Timestamps/Versions only for rows that had one,
+// which silently shifted every later row's values out of alignment with
+// SeriesList whenever an earlier row lacked a timestamp or version.
type SeriesData struct {
- SeriesList pbv1.SeriesList
- Fields FieldResultList
- Timestamps []int64
- Versions []int64
+ SeriesList pbv1.SeriesList
+ Fields FieldResultList
+ Timestamps []int64
+ Versions []int64
+ TimestampSet []bool
+ VersionSet []bool
}
// IndexDB is the interface of index database.
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000001.snp b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000001.snp
new file mode 100644
index 000000000..2aa84e988
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000001.snp differ
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.seg b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.seg
new file mode 100644
index 000000000..74b6561b0
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.seg differ
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.snp b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.snp
new file mode 100644
index 000000000..57e335bae
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/indexmode/sidx/000000000002.snp differ
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000001.snp b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000001.snp
new file mode 100644
index 000000000..efb03293a
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000001.snp differ
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.seg b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.seg
new file mode 100644
index 000000000..8e0e605d5
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.seg differ
diff --git a/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.snp b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.snp
new file mode 100644
index 000000000..b7c7011e4
Binary files /dev/null and b/banyand/internal/storage/testdata/nidx03_fixture/normal/sidx/000000000002.snp differ
diff --git a/banyand/internal/storage/tsdb.go b/banyand/internal/storage/tsdb.go
index a2204d6b2..b54f8d950 100644
--- a/banyand/internal/storage/tsdb.go
+++ b/banyand/internal/storage/tsdb.go
@@ -33,7 +33,7 @@ import (
"github.com/apache/skywalking-banyandb/banyand/observability"
obsservice "github.com/apache/skywalking-banyandb/banyand/observability/services"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ idxmetrics "github.com/apache/skywalking-banyandb/pkg/index/metrics"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -223,9 +223,9 @@ func OpenTSDB[T TSTable, O any](ctx context.Context, opts TSDBOpts[T, O], cache
clock, _ := timestamp.GetClock(ctx)
scheduler := timestamp.NewScheduler(l, clock)
- var indexMetrics *inverted.Metrics
+ var indexMetrics *idxmetrics.Metrics
if opts.StorageMetricsFactory != nil {
- indexMetrics = inverted.NewMetrics(opts.StorageMetricsFactory, common.SegLabelNames()...)
+ indexMetrics = idxmetrics.NewMetrics(opts.StorageMetricsFactory, common.SegLabelNames()...)
}
var sc Cache
if cache != nil {
diff --git a/banyand/internal/storage/tsdb_test.go b/banyand/internal/storage/tsdb_test.go
index 634a142a6..1f1d7b334 100644
--- a/banyand/internal/storage/tsdb_test.go
+++ b/banyand/internal/storage/tsdb_test.go
@@ -42,6 +42,19 @@ import (
const group = "test"
+// staticTestLease is a trivial always-valid RootLease for tests that need a
+// non-nil lease to satisfy native.Owner's "ownership is an executable
+// invariant" but don't exercise lease lifecycle (validation failure,
+// revocation) themselves. Unlike testRootLease it implements no
+// RootLeaseRevoker, so closing a database holding one never mutates shared
+// state -- safe to reuse the same value across repeated open/close cycles
+// (for example a reopen-existing-TSDB test) within one test, unlike
+// testRootLease, which permanently revokes itself on the first Close.
+type staticTestLease struct{}
+
+func (staticTestLease) Validate() error { return nil }
+func (staticTestLease) ValidatePath(string) error { return nil }
+
type testRootLease struct{ revoked atomic.Bool }
func (l *testRootLease) Validate() error {
@@ -116,6 +129,7 @@ func TestOpenTSDB(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -150,6 +164,7 @@ func TestOpenTSDB(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -201,6 +216,7 @@ func TestOpenTSDB(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -257,6 +273,7 @@ func TestSelectSegmentsRetention(t *testing.T) {
dir, defFn := test.Space(require.New(t))
t.Cleanup(defFn)
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -328,6 +345,7 @@ func TestTakeFileSnapshot(t *testing.T) {
snapshotDir := filepath.Join(dir, "snapshot")
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -375,6 +393,7 @@ func TestTakeFileSnapshot(t *testing.T) {
snapshotDir := filepath.Join(dir, "snapshot")
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -409,6 +428,7 @@ func TestTakeFileSnapshot(t *testing.T) {
defer defFn()
opts := TSDBOpts[*SnapshotMockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -448,6 +468,7 @@ func TestEpochSegmentCleanupOnOpen(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -509,6 +530,7 @@ func TestHalfBornSegmentCleanupOnOpen(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -585,6 +607,7 @@ func TestTSDBCollect(t *testing.T) {
}
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -664,6 +687,7 @@ func TestCollectWithPartialClosedSegments(t *testing.T) {
}
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -797,6 +821,7 @@ func TestTSDBOpen_LockReleasedAfterFailedOpen(t *testing.T) {
newTestSegmentSkeleton(t, dir, "1.3.0")
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -840,6 +865,7 @@ func TestTSDBOpen_LockReleasedAfterRootLeaseFactoryFailure(t *testing.T) {
defer defFn()
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -876,6 +902,7 @@ func TestTSDBCloseRevokesRootLeaseBeforeReleasingLock(t *testing.T) {
defer defFn()
lease := &testRootLease{}
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -905,6 +932,7 @@ func TestTSDBOpen_RejectsIncompatibleSegment(t *testing.T) {
newTestSegmentSkeleton(t, dir, "1.3.0")
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
diff --git a/banyand/internal/storage/visitor_test.go b/banyand/internal/storage/visitor_test.go
index 1c1c7fb6f..a6c40d6ef 100644
--- a/banyand/internal/storage/visitor_test.go
+++ b/banyand/internal/storage/visitor_test.go
@@ -100,6 +100,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB with single shard
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -157,6 +158,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB with multiple shards
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -237,6 +239,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
@@ -329,6 +332,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -379,6 +383,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 3},
@@ -430,6 +435,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB with hour-based segments
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: HOUR, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 1},
@@ -502,6 +508,7 @@ func TestVisitSegmentsInTimeRange(t *testing.T) {
// Create TSDB with daily segments
opts := TSDBOpts[*MockTSTable, any]{
+ RootLease: staticTestLease{},
Location: dir,
SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
TTL: IntervalRule{Unit: DAY, Num: 7},
diff --git a/banyand/measure/cache_benchmark_test.go b/banyand/measure/cache_benchmark_test.go
index fb1ae7839..3ae9206fd 100644
--- a/banyand/measure/cache_benchmark_test.go
+++ b/banyand/measure/cache_benchmark_test.go
@@ -38,7 +38,9 @@ import (
"github.com/apache/skywalking-banyandb/banyand/protector"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/query/model"
@@ -144,6 +146,9 @@ func openDatabase(b *testing.B, path string, cache storage.Cache) storage.TSDB[*
TSTableCreator: newTSTable,
SegmentInterval: ir,
TTL: ir,
+ RootLeaseFactory: func(lock fs.File, root string) (storage.RootLease, error) {
+ return native.NewFileRootLease(lock, root)
+ },
Option: option{
mergePolicy: newMergePolicy(math.MaxInt32, math.MaxFloat64, run.Bytes(math.MaxInt64)),
flushTimeout: time.Hour,
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index bddcfb871..fd873b5e3 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -44,6 +44,8 @@ import (
"github.com/apache/skywalking-banyandb/banyand/protector"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/banyand/queue/pub"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/meter"
resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
@@ -765,9 +767,12 @@ func (s *supplier) OpenDB(groupSchema *commonv1.Group) (resourceSchema.DB, error
}
group := groupSchema.Metadata.Name
opts := storage.TSDBOpts[*tsTable, option]{
- ShardNum: res.ResourceOpts.ShardNum,
- Location: path.Join(s.path, group),
- TSTableCreator: newTSTable,
+ ShardNum: res.ResourceOpts.ShardNum,
+ Location: path.Join(s.path, group),
+ TSTableCreator: newTSTable,
+ RootLeaseFactory: func(lock fs.File, root string) (storage.RootLease, error) {
+ return native.NewFileRootLease(lock, root)
+ },
TableMetrics: metrics,
SegmentInterval: storage.MustToIntervalRule(res.ResourceOpts.SegmentInterval),
TTL: storage.MustToIntervalRule(res.ResourceOpts.Ttl),
diff --git a/banyand/measure/migration_indexmode_copy.go b/banyand/measure/migration_indexmode_copy.go
index 64aa17074..3ce7dc7d9 100644
--- a/banyand/measure/migration_indexmode_copy.go
+++ b/banyand/measure/migration_indexmode_copy.go
@@ -33,15 +33,16 @@ import (
"github.com/apache/skywalking-banyandb/banyand/internal/migration"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
-// Reserved bluge stored-field names written by the inverted store's toDoc
-// (pkg/index/inverted/inverted.go:62-65). They are not tags and must be
-// handled explicitly rather than treated as tag fields.
+// Reserved stored-field names the series index writes (NIDX-03 §2.2;
+// storage.EncodeSeriesDocument). They are not tags and must be handled
+// explicitly rather than treated as tag fields.
const (
imDocIDField = "_id"
imTimestampField = "_timestamp"
@@ -129,7 +130,15 @@ func readIndexModeDocsNative(ctx context.Context, sidxDir string, ruleByID map[u
var timeless []uint64
tagNames := collectTagNames(schemasBySubject)
missingRules := map[uint32]int{}
- walkErr := inverted.ReadOnlyWalkDocuments(ctx, sidxDir, func(source inverted.StoredDocument) error {
+ generation, openErr := native.OpenReadOnlyGeneration(sidxDir)
+ if openErr != nil {
+ if errors.Is(openErr, native.ErrNoSnapshot) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("open sidx %s: %w", sidxDir, openErr)
+ }
+ defer func() { _ = generation.Close() }()
+ walkErr := generation.VisitLiveDocuments(ctx, func(source native.StoredDocument) error {
document, rebuildErr := rebuildOneDoc(source, ruleByID, schemasBySubject, tagNames, missingRules)
if rebuildErr != nil {
return fmt.Errorf("rebuild doc in %s: %w", sidxDir, rebuildErr)
@@ -142,9 +151,6 @@ func readIndexModeDocsNative(ctx context.Context, sidxDir string, ruleByID map[u
return nil
})
if walkErr != nil {
- if errors.Is(walkErr, inverted.ErrNoCommittedIndex) {
- return nil, nil
- }
return nil, fmt.Errorf("walk sidx %s: %w", sidxDir, walkErr)
}
if len(timeless) > 0 {
@@ -225,41 +231,105 @@ func appendRegeneratedEntityFields(fields []index.Field, series *pbv1.Series, sc
// ── Target idx store pool ────────────────────────────────────────────────────.
-// targetIdxStore lazily opens one inverted store per target sidx path
-// (BatchWaitSec:0) and closes them all at the end, so multiple source segs
-// feeding the same target sidx share a single writer. Copied (cross-package,
-// unexported) from stream/migration_element_index.go:126-158.
+// indexModeLfs performs this tool's own lock-file management, matching
+// pkg/index/native's own package-level fileSystem and the union-sidx
+// builder's.
+var indexModeLfs = fs.NewLocalFileSystem()
+
+// indexModeLockFilename is the lock file targetIdxStore creates (one per
+// target segment, beside its sidx directory) to back the native.FileRootLease
+// each target owner requires; it is removed once that owner closes.
+const indexModeLockFilename = "lock"
+
+// targetIdxStore lazily opens one native owner per target sidx path and
+// closes them all at the end, so multiple source segs feeding the same
+// target sidx share a single writer. Adapted (cross-package, unexported)
+// from stream/migration_element_index.go's original inverted-store pool.
type targetIdxStore struct {
- stores map[string]index.SeriesStore
+ owners map[string]*native.Owner
+ locks map[string]fs.File
}
func newTargetIdxStore() *targetIdxStore {
- return &targetIdxStore{stores: map[string]index.SeriesStore{}}
+ return &targetIdxStore{owners: map[string]*native.Owner{}, locks: map[string]fs.File{}}
}
-func (t *targetIdxStore) get(path string) (index.SeriesStore, error) {
- if s, ok := t.stores[path]; ok {
- return s, nil
+func (t *targetIdxStore) get(path string) (*native.Owner, error) {
+ if o, ok := t.owners[path]; ok {
+ return o, nil
}
if err := os.MkdirAll(path, storage.DirPerm); err != nil {
return nil, fmt.Errorf("mkdir idx %s: %w", path, err)
}
- s, err := inverted.NewStore(inverted.StoreOpts{Path: path, BatchWaitSec: 0})
+ // native.FileRootLease requires its owner's Path to be a proper
+ // subdirectory of the lease root, so the lock lives one level up from
+ // the sidx directory itself (the target segment directory), matching a
+ // live TSDB's own lock-beside-segment layout.
+ leaseRoot := filepath.Dir(path)
+ lockPath := filepath.Join(leaseRoot, indexModeLockFilename)
+ lock, lockErr := indexModeLfs.CreateLockFile(lockPath, storage.FilePerm)
+ if lockErr != nil {
+ return nil, fmt.Errorf("create idx lock %s: %w", lockPath, lockErr)
+ }
+ lease, leaseErr := native.NewFileRootLease(lock, leaseRoot)
+ if leaseErr != nil {
+ _ = lock.Close()
+ return nil, fmt.Errorf("create idx root lease %s: %w", path, leaseErr)
+ }
+ // NewOwner is a synchronous constructor and has no context-bearing API.
+ //nolint:contextcheck // construction does not perform cancellable I/O
+ owner, ownerErr := native.NewOwner(native.OwnerOptions{Lease: lease, Path: path, IdentifierDocValues: true})
+ if ownerErr != nil {
+ _ = lock.Close()
+ return nil, fmt.Errorf("open idx store %s: %w", path, ownerErr)
+ }
+ t.owners[path] = owner
+ t.locks[path] = lock
+ return owner, nil
+}
+
+// writeDocs upserts docs into the target sidx at path (opening it on first
+// use), through the same series document mapping the live series index uses
+// (storage.EncodeSeriesDocument), and blocks until the batch is durable --
+// matching the previous release's inverted.NewStore(StoreOpts{BatchWaitSec:
+// 0}) synchronous-write contract, which callers (including the verify/
+// analyze readers that immediately re-open the target sidx) depend on.
+func (t *targetIdxStore) writeDocs(path string, docs index.Documents) error {
+ owner, err := t.get(path)
if err != nil {
- return nil, fmt.Errorf("open idx store %s: %w", path, err)
+ return err
}
- t.stores[path] = s
- return s, nil
+ nativeDocs := make([]native.Document, len(docs))
+ for i := range docs {
+ nd, encErr := storage.EncodeSeriesDocument(docs[i])
+ if encErr != nil {
+ return fmt.Errorf("encode doc for %s: %w", path, encErr)
+ }
+ nativeDocs[i] = nd
+ }
+ done := make(chan error, 1)
+ if err := owner.Batch(context.Background(), native.Batch{
+ Documents: nativeDocs,
+ PersistentCallback: func(err error) { done <- err },
+ }); err != nil {
+ return err
+ }
+ return <-done
}
func (t *targetIdxStore) closeAll() error {
var firstErr error
- for path, s := range t.stores {
- if err := s.Close(); err != nil && firstErr == nil {
+ for path, o := range t.owners {
+ if err := o.Close(); err != nil && firstErr == nil {
firstErr = fmt.Errorf("close idx store %s: %w", path, err)
}
}
- t.stores = map[string]index.SeriesStore{}
+ for path, lock := range t.locks {
+ _ = lock.Close()
+ _ = os.Remove(filepath.Join(filepath.Dir(path), indexModeLockFilename))
+ }
+ t.owners = map[string]*native.Owner{}
+ t.locks = map[string]fs.File{}
return firstErr
}
@@ -305,17 +375,16 @@ func copyIndexModeSlowDocs(ctx context.Context, srcSidxDirs []string, docsByDir
if ctx.Err() != nil {
return rows, ctx.Err()
}
- store, getErr := stores.get(dstSidx)
- if getErr != nil {
- return rows, getErr
- }
- batch := index.Batch{Documents: make(index.Documents, 0, len(m))}
+ docs := make(index.Documents, 0, len(m))
for _, d := range m {
- batch.Documents = append(batch.Documents, d)
+ docs = append(docs, d)
rows++
}
- if updateErr := store.UpdateSeriesBatch(batch); updateErr != nil {
- return rows, fmt.Errorf("write target sidx %s: %w", dstSidx, updateErr)
+ // writeDocs's lazy native.NewOwner construction is synchronous and has
+ // no context-bearing API (see the nolint on targetIdxStore.get).
+ //nolint:contextcheck
+ if writeErr := stores.writeDocs(dstSidx, docs); writeErr != nil {
+ return rows, fmt.Errorf("write target sidx %s: %w", dstSidx, writeErr)
}
segDir := filepath.Join(dstGroupRoot, segNameByPath[dstSidx])
if _, metaErr := writeIndexModeSegMetadata(segDir, ir); metaErr != nil {
@@ -367,7 +436,7 @@ func copyIndexModeGroup(ctx context.Context, in migration.EntryGroupInput,
}
stores := newTargetIdxStore()
- // Close() flushes buffered batches and lands bluge's own snapshot; a close
+ // Close() flushes buffered batches and lands the native owner's snapshot; a close
// failure means an incomplete sidx on disk, so it must surface even on the
// happy path.
defer func() {
@@ -472,7 +541,7 @@ func copyIndexModeGroup(ctx context.Context, in migration.EntryGroupInput,
}
// Flush the slow-path stores before counting segments so every target sidx
- // (and its bluge snapshot) is on disk. closeAll is idempotent; the deferred
+ // (and its native snapshot) is on disk. closeAll is idempotent; the deferred
// call becomes a no-op.
if closeErr := stores.closeAll(); closeErr != nil {
return res, closeErr
@@ -693,7 +762,7 @@ func readAllIndexModeDocs(ctx context.Context, dirs []string, ruleByID map[uint3
) ([]index.Document, error) {
var out []index.Document
for _, dir := range dirs {
- docs, err := readIndexModeDocs(ctx, dir, ruleByID, schemasBySubject)
+ docs, err := readIndexModeDocsNative(ctx, dir, ruleByID, schemasBySubject)
if err != nil {
return nil, err
}
@@ -731,7 +800,7 @@ func segKeyedDigests(ctx context.Context, dirs []string, ruleByID map[uint32]ind
}
var segCache alignedSegCache
for _, dir := range dirs {
- docs, readErr := readIndexModeDocs(ctx, dir, ruleByID, schemasBySubject)
+ docs, readErr := readIndexModeDocsNative(ctx, dir, ruleByID, schemasBySubject)
if readErr != nil {
return nil, readErr
}
diff --git a/banyand/measure/migration_indexmode_copy_test.go b/banyand/measure/migration_indexmode_copy_test.go
index 9ae94d8ea..500fbbea6 100644
--- a/banyand/measure/migration_indexmode_copy_test.go
+++ b/banyand/measure/migration_indexmode_copy_test.go
@@ -19,12 +19,15 @@ package measure
import (
"context"
+ "fmt"
"os"
"path/filepath"
+ "strings"
"testing"
"time"
"github.com/blugelabs/bluge"
+ "github.com/blugelabs/bluge/search"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -186,6 +189,112 @@ func fieldByMarshaledName(d index.Document, name string) (index.Field, bool) {
return index.Field{}, false
}
+// legacyStoredDocument adapts *bluge's own search.DocumentMatch to
+// storedFieldDocument (migration_verify.go): an explicit wrapper, not a bare
+// structural conversion, because bluge's VisitStoredFields takes its own
+// named segment.StoredFieldVisitor function type rather than the plain
+// func(string, []byte) bool storedFieldDocument expects.
+type legacyStoredDocument struct {
+ match *search.DocumentMatch
+}
+
+func (d legacyStoredDocument) VisitStoredFields(visit func(name string, value []byte) bool) error {
+ return d.match.VisitStoredFields(func(field string, value []byte) bool {
+ return visit(field, value)
+ })
+}
+
+// readIndexModeDocsLegacy is the previous release's own reader
+// (pkg/index/inverted.NewStore's bluge.OpenReader), restored here for TEST
+// verification only (L4): it reuses rebuildOneDoc -- the same rebuild logic
+// readIndexModeDocsNative calls -- so the only thing this function exercises
+// independently of production is the READER half, proving the native
+// writer's on-disk output is byte-compatible with, and genuinely readable
+// by, a previous-release node, not merely round-trippable through the same
+// native code that wrote it.
+func readIndexModeDocsLegacy(ctx context.Context, sidxDir string, ruleByID map[uint32]indexRuleInfo,
+ schemasBySubject map[string]*measureSchemaInfo,
+) ([]index.Document, error) {
+ reader, openErr := bluge.OpenReader(bluge.DefaultConfig(sidxDir))
+ if openErr != nil {
+ if strings.Contains(openErr.Error(), "unable to find a usable snapshot") {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("open sidx reader %s: %w", sidxDir, openErr)
+ }
+ defer func() { _ = reader.Close() }()
+ matches, searchErr := reader.Search(ctx, bluge.NewAllMatches(bluge.NewMatchAllQuery()))
+ if searchErr != nil {
+ return nil, fmt.Errorf("search sidx %s: %w", sidxDir, searchErr)
+ }
+ var documents []index.Document
+ tagNames := collectTagNames(schemasBySubject)
+ missingRules := map[uint32]int{}
+ for {
+ match, nextErr := matches.Next()
+ if nextErr != nil {
+ return nil, fmt.Errorf("iterate sidx %s: %w", sidxDir, nextErr)
+ }
+ if match == nil {
+ break
+ }
+ document, rebuildErr := rebuildOneDoc(legacyStoredDocument{match: match}, ruleByID, schemasBySubject, tagNames, missingRules)
+ if rebuildErr != nil {
+ return nil, fmt.Errorf("rebuild doc in %s: %w", sidxDir, rebuildErr)
+ }
+ documents = append(documents, document)
+ }
+ return documents, nil
+}
+
+// indexModeDocsByID converts docs into a map keyed by DocID for
+// order-independent comparison.
+func indexModeDocsByID(docs []index.Document) map[uint64]index.Document {
+ out := make(map[uint64]index.Document, len(docs))
+ for _, d := range docs {
+ out[d.DocID] = d
+ }
+ return out
+}
+
+// indexModeFieldMap flattens a document's Fields into a
+// marshaled-key -> raw-bytes map for order-independent comparison. Every
+// field rebuildOneDoc produces carries a *index.BytesTermValue term.
+func indexModeFieldMap(t *testing.T, fields []index.Field) map[string][]byte {
+ t.Helper()
+ out := make(map[string][]byte, len(fields))
+ for i := range fields {
+ term, ok := fields[i].GetTerm().(*index.BytesTermValue)
+ require.True(t, ok, "field %q has unexpected term type %T", fields[i].Key.Marshal(), fields[i].GetTerm())
+ out[fields[i].Key.Marshal()] = term.Value
+ }
+ return out
+}
+
+// requireIndexModeDocsReadableByLegacyReader is the L4 regression helper:
+// it re-reads sidxDir with the previous release's own bluge reader
+// (readIndexModeDocsLegacy) and asserts its output matches nativeDocs
+// field-for-field, so a test that only asserted against
+// readIndexModeDocsNative's output also proves the migrated sidx opens with
+// a reader independent of the one production itself uses to verify it.
+func requireIndexModeDocsReadableByLegacyReader(t *testing.T, sidxDir string, ruleByID map[uint32]indexRuleInfo,
+ schemasBySubject map[string]*measureSchemaInfo, nativeDocs []index.Document,
+) {
+ t.Helper()
+ legacyDocs, err := readIndexModeDocsLegacy(context.Background(), sidxDir, ruleByID, schemasBySubject)
+ require.NoError(t, err)
+ want := indexModeDocsByID(nativeDocs)
+ got := indexModeDocsByID(legacyDocs)
+ require.Len(t, got, len(want), "the previous release's own reader must see the same document count")
+ for id, w := range want {
+ g, ok := got[id]
+ require.True(t, ok, "previous-release reader missing doc %d", id)
+ require.Equal(t, w.Timestamp, g.Timestamp, "doc %d timestamp", id)
+ require.Equal(t, w.Version, g.Version, "doc %d version", id)
+ require.Equal(t, indexModeFieldMap(t, w.Fields), indexModeFieldMap(t, g.Fields), "doc %d fields", id)
+ }
+}
+
func TestReadAndRebuildIndexModeDoc(t *testing.T) {
dir := t.TempDir()
const ruleID = uint32(201559343)
@@ -211,7 +320,7 @@ func TestReadAndRebuildIndexModeDoc(t *testing.T) {
}
ruleByID := map[uint32]indexRuleInfo{ruleID: {Analyzer: index.AnalyzerURL, NoSort: false}}
- docs, err := readIndexModeDocs(context.Background(), dir, ruleByID, schemasBySubject)
+ docs, err := readIndexModeDocsNative(context.Background(), dir, ruleByID, schemasBySubject)
require.NoError(t, err)
require.Len(t, docs, 1)
d := docs[0]
@@ -276,7 +385,7 @@ func TestReadAndRebuildIndexModeDoc_SkipsIndexedEntityTag(t *testing.T) {
},
}
- docs, err := readIndexModeDocs(context.Background(), dir, map[uint32]indexRuleInfo{}, schemasBySubject)
+ docs, err := readIndexModeDocsNative(context.Background(), dir, map[uint32]indexRuleInfo{}, schemasBySubject)
require.NoError(t, err)
require.Len(t, docs, 1)
@@ -289,7 +398,7 @@ func TestReadAndRebuildIndexModeDoc_SkipsIndexedEntityTag(t *testing.T) {
}
func TestReadIndexModeDocs_EmptySidx(t *testing.T) {
- docs, err := readIndexModeDocs(context.Background(), t.TempDir(), map[uint32]indexRuleInfo{}, nil)
+ docs, err := readIndexModeDocsNative(context.Background(), t.TempDir(), map[uint32]indexRuleInfo{}, nil)
require.NoError(t, err)
require.Empty(t, docs)
}
@@ -374,11 +483,17 @@ func svcSchemas() map[string]*measureSchemaInfo {
}
}
-// countSidxDocs opens a sidx read-only and returns the number of docs.
+// countSidxDocs opens a sidx read-only and returns the number of docs. It
+// also cross-checks the native reader's output against the previous
+// release's own bluge reader (L4), so every caller -- most read a
+// migration's native-written target sidx -- proves the migrated output is
+// independently readable, not merely round-trippable through the same
+// native code production uses to verify it.
func countSidxDocs(t *testing.T, sidxDir string) int {
t.Helper()
- docs, err := readIndexModeDocs(context.Background(), sidxDir, map[uint32]indexRuleInfo{}, svcSchemas())
+ docs, err := readIndexModeDocsNative(context.Background(), sidxDir, map[uint32]indexRuleInfo{}, svcSchemas())
require.NoError(t, err)
+ requireIndexModeDocsReadableByLegacyReader(t, sidxDir, map[uint32]indexRuleInfo{}, svcSchemas(), docs)
return len(docs)
}
@@ -393,7 +508,27 @@ func TestTargetIdxStore_GetSamePathSharesWriter(t *testing.T) {
require.Same(t, s1, s2, "same path must share one writer")
require.NoError(t, pool.closeAll())
- require.Empty(t, pool.stores, "closeAll must clear the pool")
+ require.Empty(t, pool.owners, "closeAll must clear the pool")
+}
+
+// TestTargetIdxStore_GetSurvivesStaleLockFromPriorCrash is part of L8: a
+// tool crash leaves indexModeLockFilename ("lock", beside the target sidx
+// directory) on disk, since it is removed only in a clean closeAll. The
+// next run's targetIdxStore.get must not be blocked by that stale file --
+// CreateLockFile truncates and re-flocks rather than requiring the path be
+// absent, and an OS-level flock is released when the crashed process exits
+// regardless of whether Close ran, so a leftover lock file with no live
+// holder must never fail a fresh get.
+func TestTargetIdxStore_GetSurvivesStaleLockFromPriorCrash(t *testing.T) {
+ segDir := t.TempDir()
+ path := filepath.Join(segDir, directCopySidxDirName)
+ require.NoError(t, os.MkdirAll(path, storage.DirPerm))
+ require.NoError(t, os.WriteFile(filepath.Join(segDir, indexModeLockFilename), []byte("stale"), storage.FilePerm))
+
+ pool := newTargetIdxStore()
+ _, err := pool.get(path)
+ require.NoError(t, err, "a stale lock file from a prior crash must not block the next run")
+ require.NoError(t, pool.closeAll())
}
// readDocsByDir reads every dir's index-mode docs into the docsByDir map the
@@ -402,7 +537,7 @@ func readDocsByDir(t *testing.T, dirs []string) map[string][]index.Document {
t.Helper()
out := make(map[string][]index.Document, len(dirs))
for _, dir := range dirs {
- docs, err := readIndexModeDocs(context.Background(), dir, map[uint32]indexRuleInfo{}, svcSchemas())
+ docs, err := readIndexModeDocsNative(context.Background(), dir, map[uint32]indexRuleInfo{}, svcSchemas())
require.NoError(t, err)
out[dir] = docs
}
@@ -437,9 +572,10 @@ func TestSlowPath_MergeKeepsMaxVersion(t *testing.T) {
require.NoError(t, stores.closeAll())
targetSidx := filepath.Join(dst, "seg-20260621", directCopySidxDirName)
- docs, err := readIndexModeDocs(context.Background(), targetSidx, map[uint32]indexRuleInfo{}, svcSchemas())
+ docs, err := readIndexModeDocsNative(context.Background(), targetSidx, map[uint32]indexRuleInfo{}, svcSchemas())
require.NoError(t, err)
require.Len(t, docs, 1)
+ requireIndexModeDocsReadableByLegacyReader(t, targetSidx, map[uint32]indexRuleInfo{}, svcSchemas(), docs)
require.Equal(t, int64(9), docs[0].Version, "highest version must win")
pf, ok := fieldByMarshaledName(docs[0], "properties")
require.True(t, ok)
@@ -544,9 +680,10 @@ func TestCopyIndexModeGroup_MergeAcrossSourcesKeepsMaxVersion(t *testing.T) {
require.Equal(t, int64(1), res.Rows, "same series across sources collapses to one row")
targetSidx := filepath.Join(dst, "seg-20260621", directCopySidxDirName)
- docs, err := readIndexModeDocs(context.Background(), targetSidx, map[uint32]indexRuleInfo{}, svcSchemas())
+ docs, err := readIndexModeDocsNative(context.Background(), targetSidx, map[uint32]indexRuleInfo{}, svcSchemas())
require.NoError(t, err)
require.Len(t, docs, 1)
+ requireIndexModeDocsReadableByLegacyReader(t, targetSidx, map[uint32]indexRuleInfo{}, svcSchemas(), docs)
require.Equal(t, int64(9), docs[0].Version, "highest version must win across sources")
pf, ok := fieldByMarshaledName(docs[0], "properties")
require.True(t, ok)
@@ -596,9 +733,10 @@ func TestCopyIndexModeGroup_ByteCopyTargetAlsoReceivesSplitSource_KeepsMaxVersio
require.Zero(t, res.Bytes, "no source exclusively owns a target seg, so nothing byte-copies")
h1Sidx := filepath.Join(dst, "seg-2026062101", directCopySidxDirName)
- docs, err := readIndexModeDocs(context.Background(), h1Sidx, map[uint32]indexRuleInfo{}, svcSchemas())
+ docs, err := readIndexModeDocsNative(context.Background(), h1Sidx, map[uint32]indexRuleInfo{}, svcSchemas())
require.NoError(t, err)
require.Len(t, docs, 1, "hour-01 holds exactly the shared series")
+ requireIndexModeDocsReadableByLegacyReader(t, h1Sidx, map[uint32]indexRuleInfo{}, svcSchemas(), docs)
require.Equal(t, int64(100), docs[0].Version, "max version must survive across byte-copy + split source")
pf, ok := fieldByMarshaledName(docs[0], "properties")
require.True(t, ok)
@@ -707,12 +845,15 @@ func TestCopyIndexModeGroup_TimestampZeroSurfacesError(t *testing.T) {
// ── Phase 3 / 4: value digest, verify, analyze tests ─────────────────────────.
-// readOneDoc reads exactly one rebuilt doc from a sidx dir, failing otherwise.
+// readOneDoc reads exactly one rebuilt doc from a sidx dir, failing
+// otherwise, and cross-checks it against the previous release's own bluge
+// reader (L4; see countSidxDocs).
func readOneDoc(t *testing.T, sidxDir string, schemas map[string]*measureSchemaInfo) index.Document {
t.Helper()
- docs, err := readIndexModeDocs(context.Background(), sidxDir, map[uint32]indexRuleInfo{}, schemas)
+ docs, err := readIndexModeDocsNative(context.Background(), sidxDir, map[uint32]indexRuleInfo{}, schemas)
require.NoError(t, err)
require.Len(t, docs, 1)
+ requireIndexModeDocsReadableByLegacyReader(t, sidxDir, map[uint32]indexRuleInfo{}, schemas, docs)
return docs[0]
}
@@ -1353,8 +1494,9 @@ func TestIndexModeMigration_SplitAndMergeRebuild(t *testing.T) {
"hour 05 holds svc-c")
// Max-version merge: the surviving svc-a doc must carry version 5 + p-a-v2.
- docs, err := readIndexModeDocs(context.Background(), hour01, e2eRuleByID(), e2eSchemasBySubject())
+ docs, err := readIndexModeDocsNative(context.Background(), hour01, e2eRuleByID(), e2eSchemasBySubject())
require.NoError(t, err)
+ requireIndexModeDocsReadableByLegacyReader(t, hour01, e2eRuleByID(), e2eSchemasBySubject(), docs)
var svcADoc *index.Document
for i := range docs {
var series pbv1.Series
@@ -1467,13 +1609,15 @@ func TestIndexModeMigration_SearchabilityPreserved(t *testing.T) {
}
}
-// countSidxDocsWith opens a sidx read-only with explicit rule/schema tables and
-// returns the rebuilt doc count.
+// countSidxDocsWith opens a sidx read-only with explicit rule/schema tables,
+// returns the rebuilt doc count, and cross-checks against the previous
+// release's own bluge reader (L4; see countSidxDocs).
func countSidxDocsWith(t *testing.T, sidxDir string, ruleByID map[uint32]indexRuleInfo,
schemas map[string]*measureSchemaInfo,
) int {
t.Helper()
- docs, err := readIndexModeDocs(context.Background(), sidxDir, ruleByID, schemas)
+ docs, err := readIndexModeDocsNative(context.Background(), sidxDir, ruleByID, schemas)
require.NoError(t, err)
+ requireIndexModeDocsReadableByLegacyReader(t, sidxDir, ruleByID, schemas, docs)
return len(docs)
}
diff --git a/banyand/measure/migration_indexmode_native_source_test.go b/banyand/measure/migration_indexmode_native_source_test.go
index 1d9d44fca..9572f3142 100644
--- a/banyand/measure/migration_indexmode_native_source_test.go
+++ b/banyand/measure/migration_indexmode_native_source_test.go
@@ -37,6 +37,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -49,9 +50,13 @@ const (
indexModeSourceReadFile = "migration_indexmode_copy.go"
// nativeVisitorPackage and nativeVisitorFunc name the BanyanDB-owned
- // read-only document visitor issue #14010 introduces.
- nativeVisitorPackage = "github.com/apache/skywalking-banyandb/pkg/index/inverted"
- nativeVisitorFunc = "ReadOnlyWalkDocuments"
+ // read-only document visitor the series index cutover (NIDX-03) moved
+ // this file's source read onto: a native.ReadOnlyGeneration opened
+ // directly, instead of the pkg/index/inverted wrapper issue #14010
+ // introduced (itself already nativeice-backed, but still a
+ // pkg/index/inverted dependency the series-index cutover retires).
+ nativeVisitorPackage = "github.com/apache/skywalking-banyandb/pkg/index/native"
+ nativeVisitorFunc = "OpenReadOnlyGeneration"
// retiredReaderEntryPoint is the call that opens a retired third-party
// index reader directly on a source directory. The whole point of the
@@ -63,10 +68,15 @@ const (
iceFooterLength = 60
// nativeSourceTagName is the stored, non-indexed tag the seeded sources
- // carry, and publishedSeriesLimit bounds a destination lookup so a stray
- // duplicate is reported rather than silently truncated away.
- nativeSourceTagName = "properties"
- publishedSeriesLimit = 100
+ // carry.
+ nativeSourceTagName = "properties"
+
+ // nativeSourceIdentifierField, nativeSourceTimestampField and
+ // nativeSourceVersionField are the reserved stored fields a series document
+ // records its identity, timestamp and version under.
+ nativeSourceIdentifierField = "_id"
+ nativeSourceTimestampField = "_timestamp"
+ nativeSourceVersionField = "_version"
)
// TestIndexModeCopySourceReadReachesNativeVisitor is the structural half of the
@@ -219,53 +229,52 @@ func nativeSourceIdentity(t *testing.T, declared nativeSourceSeries) []byte {
return append([]byte(nil), series.Buffer...)
}
-// observedSeries is what the independent destination oracle reports for one
-// published series: its timestamp, its version and its stored tag values. It is
-// read back through the retained series-store query path, which this milestone
-// does not touch, so it cannot agree with a faulty source read by construction.
+// observedSeries is what the destination oracle reports for one published
+// series: its timestamp, its version and its stored tag values. It is read
+// back through an exact identifier selection on the destination's term
+// dictionary, a different read path from the live-document walk the copy reads
+// its sources with.
type observedSeries struct {
tagValues []string
timestamp int64
version int64
}
-// openPublishedDestination opens a published destination sidx through the
-// retained series-store surface, which this milestone does not touch, so what
-// it reports cannot agree with a faulty source read by construction.
-func openPublishedDestination(t *testing.T, sidxDir string) index.SeriesStore {
- t.Helper()
- store, err := inverted.NewStore(inverted.StoreOpts{Path: sidxDir, BatchWaitSec: 0})
- require.NoError(t, err)
- return store
-}
-
// publishedSeriesFor looks one identity up in a published destination and
// reports what it holds, or reports that the destination holds no live document
// under that identity. The lookup is an exact one so that the answer for one
// series can never be contaminated by another's.
-func publishedSeriesFor(t *testing.T, store index.SeriesStore, identity []byte) (observedSeries, bool) {
+func publishedSeriesFor(t *testing.T, sidxDir string, identity []byte) (observedSeries, bool) {
t.Helper()
- query, err := store.BuildQuery([]index.SeriesMatcher{
- {Type: index.SeriesMatcherTypeExact, Match: identity},
- }, nil, nil)
- require.NoError(t, err)
- documents, err := store.Search(context.Background(),
- []index.FieldKey{{TagName: nativeSourceTagName}}, query, publishedSeriesLimit)
- require.NoError(t, err)
- if len(documents) == 0 {
+ var observed observedSeries
+ matches := 0
+ selection := native.TermSelection{Field: nativeSourceIdentifierField, Terms: [][]byte{identity}}
+ require.NoError(t, native.ReadOnlySelectDocuments(context.Background(), sidxDir, selection, func(document native.StoredDocument) error {
+ matches++
+ var fieldErr error
+ visitErr := document.VisitStoredFields(func(name string, value []byte) bool {
+ switch name {
+ case nativeSourceTimestampField:
+ observed.timestamp, fieldErr = native.DecodeTimestamp(value)
+ return fieldErr == nil
+ case nativeSourceVersionField:
+ observed.version = convert.BytesToInt64(value)
+ case nativeSourceTagName:
+ observed.tagValues = append(observed.tagValues, string(value))
+ }
+ return true
+ })
+ if visitErr != nil {
+ return visitErr
+ }
+ return fieldErr
+ }))
+ if matches == 0 {
return observedSeries{}, false
}
- require.Len(t, documents, 1, "identity %q must resolve to at most one live destination document", identity)
- var values []string
- if value, stored := documents[0].Fields[nativeSourceTagName]; stored && value != nil {
- values = append(values, string(value))
- }
- sort.Strings(values)
- return observedSeries{
- timestamp: documents[0].Timestamp,
- version: documents[0].Version,
- tagValues: values,
- }, true
+ require.Equal(t, 1, matches, "identity %q must resolve to at most one live destination document", identity)
+ sort.Strings(observed.tagValues)
+ return observed, true
}
// truncateSourceSegment cuts a source segment file short of the footer its
@@ -288,8 +297,7 @@ func truncateSourceSegment(t *testing.T, sidxDir string) {
// milestone exists for: an operator migrates an index-mode measure group whose
// segment indexes were written by a released BanyanDB. Index-mode copy reads
// those sources through the native document visitor and publishes the rebuilt
-// documents through its retained destination writer, which this milestone does
-// not change.
+// documents through a native destination owner.
//
// Two source roots feed the same target segment, which is what a group
// migrated from replicated nodes looks like and what forces the copy down its
@@ -300,8 +308,7 @@ func truncateSourceSegment(t *testing.T, sidxDir string) {
//
// R2 -- index-mode copy rebuilds the two live series through the native source
// visitor, preserving each one's declared timestamp, version, stored tag
-// value and identity, while its destination stays written by the
-// retained writer. The deleted series never reaches the destination.
+// value and identity. The deleted series never reaches the destination.
//
// R4 -- a source whose committed segment is damaged fails the copy with the
// native typed corruption error and publishes no destination, so a
@@ -333,17 +340,13 @@ func TestE2EIndexModeCopyNativeSource(t *testing.T) {
"the two live series survive; the repeat of the first collapses onto it and the deleted one is never read")
targetSidx := filepath.Join(targetRoot, sourceSegName, directCopySidxDirName)
- publishedCount, countErr := inverted.ReadOnlyDocCount(targetSidx)
+ publishedCount, countErr := native.ReadOnlyDocCount(targetSidx)
tester.NoError(countErr)
tester.Equal(int64(2), publishedCount, "the destination must hold exactly the two live series")
- destination := openPublishedDestination(t, targetSidx)
- defer func() {
- tester.NoError(destination.Close())
- }()
for _, declaredSeries := range declared {
identity := nativeSourceIdentity(t, declaredSeries)
- observed, found := publishedSeriesFor(t, destination, identity)
+ observed, found := publishedSeriesFor(t, targetSidx, identity)
if declaredSeries.deleted {
tester.False(found, "the deleted series labeled %s must not reach the destination", declaredSeries.label)
continue
@@ -368,7 +371,7 @@ func TestE2EIndexModeCopyNativeSource(t *testing.T) {
SrcRoots: []string{corruptRoot},
Interval: interval,
}, map[uint32]indexRuleInfo{}, svcSchemas())
- tester.ErrorIs(corruptErr, inverted.ErrCorruptIndex,
+ tester.ErrorIs(corruptErr, native.ErrCorrupt,
"a damaged source must be reported as a corrupt index, not as an opaque read failure")
tester.NoDirExists(filepath.Join(corruptTarget, sourceSegName, directCopySidxDirName),
"a copy that failed on a damaged source must publish no destination sidx")
diff --git a/banyand/measure/migration_verify.go b/banyand/measure/migration_verify.go
index 9e7aa0aa6..84cf74162 100644
--- a/banyand/measure/migration_verify.go
+++ b/banyand/measure/migration_verify.go
@@ -18,7 +18,6 @@
package measure
import (
- "context"
"encoding/json"
"errors"
"fmt"
@@ -29,12 +28,11 @@ import (
"strings"
"time"
- compat "github.com/blugelabs/bluge"
-
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
@@ -189,90 +187,30 @@ func partRowCount(partID uint64, name, shardDir string, fileSystem fs.FileSystem
return p.partMetadata.TotalCount, nil
}
-// CountBlugeDocs opens the bluge index at path read-only and returns the
-// total document count. Used to spot-check the broadcast union sidx
-// that `migration copy` writes into every aligned target segment.
-func CountBlugeDocs(path string) (uint64, error) {
- reader, err := compat.OpenReader(compat.DefaultConfig(path))
+// CountSeriesIndexDocs opens the series index at path read-only (a native
+// read-only generation, never a writable owner) and returns the total live
+// document count. Used to spot-check the broadcast union sidx that
+// `migration copy` writes into every aligned target segment.
+func CountSeriesIndexDocs(path string) (uint64, error) {
+ generation, err := native.OpenReadOnlyGeneration(path)
if err != nil {
+ if errors.Is(err, native.ErrNoSnapshot) {
+ return 0, nil
+ }
return 0, err
}
- defer func() { _ = reader.Close() }()
- // Reader.Count sums per-segment doc counts in O(numSegments), not
- // O(numDocs) — important for `verify` because the union sidx is
- // broadcast into every aligned target seg of a group and we call
- // CountBlugeDocs once per seg. Iterating dmi.Next() instead would
- // scale with total docs × broadcast fan-out and dominate verify
- // runtime on production-sized datasets.
- return reader.Count()
+ defer func() { _ = generation.Close() }()
+ count, countErr := generation.VisibleDocCount()
+ if countErr != nil {
+ return 0, countErr
+ }
+ return uint64(count), nil
}
type storedFieldDocument interface {
VisitStoredFields(visit func(name string, value []byte) bool) error
}
-type retainedStoredDocument struct {
- visitStoredFields func(func(name string, value []byte) bool) error
-}
-
-func (d retainedStoredDocument) VisitStoredFields(visit func(name string, value []byte) bool) error {
- return d.visitStoredFields(visit)
-}
-
-func readIndexModeDocs(ctx context.Context, sidxDir string, ruleByID map[uint32]indexRuleInfo,
- schemasBySubject map[string]*measureSchemaInfo,
-) ([]index.Document, error) {
- reader, openErr := compat.OpenReader(compat.DefaultConfig(sidxDir))
- if openErr != nil {
- if strings.Contains(openErr.Error(), "unable to find a usable snapshot") {
- return nil, nil
- }
- return nil, fmt.Errorf("open sidx reader %s: %w", sidxDir, openErr)
- }
- defer func() { _ = reader.Close() }()
- matches, searchErr := reader.Search(ctx, compat.NewAllMatches(compat.NewMatchAllQuery()))
- if searchErr != nil {
- return nil, fmt.Errorf("search sidx %s: %w", sidxDir, searchErr)
- }
- var documents []index.Document
- var timeless []uint64
- tagNames := collectTagNames(schemasBySubject)
- missingRules := map[uint32]int{}
- for {
- match, nextErr := matches.Next()
- if nextErr != nil {
- return nil, fmt.Errorf("iterate sidx %s: %w", sidxDir, nextErr)
- }
- if match == nil {
- break
- }
- document, rebuildErr := rebuildOneDoc(retainedStoredDocument{visitStoredFields: func(visit func(name string, value []byte) bool) error {
- return match.VisitStoredFields(func(name string, value []byte) bool {
- return visit(name, value)
- })
- }}, ruleByID, schemasBySubject, tagNames, missingRules)
- if rebuildErr != nil {
- return nil, fmt.Errorf("rebuild doc in %s: %w", sidxDir, rebuildErr)
- }
- if document.Timestamp == 0 {
- timeless = append(timeless, document.DocID)
- continue
- }
- documents = append(documents, document)
- }
- if len(timeless) > 0 {
- sample := timeless
- if len(sample) > 10 {
- sample = sample[:10]
- }
- return nil, fmt.Errorf("sidx %s: %d index-mode doc(s) have a missing or undecodable _timestamp (ts==0), "+
- "which never occurs for valid data; the source is corrupt or unexpected — sample series IDs (first %d) %v",
- sidxDir, len(timeless), len(sample), sample)
- }
- warnMissingRules(sidxDir, missingRules)
- return documents, nil
-}
-
func rebuildOneDoc(source storedFieldDocument, ruleByID map[uint32]indexRuleInfo,
schemasBySubject map[string]*measureSchemaInfo, tagNames map[string]struct{}, missingRules map[uint32]int,
) (index.Document, error) {
@@ -284,8 +222,8 @@ func rebuildOneDoc(source storedFieldDocument, ruleByID map[uint32]indexRuleInfo
case imDocIDField:
entityValues = append([]byte(nil), value...)
case imTimestampField:
- if decodedTime, decodeErr := compat.DecodeDateTime(value); decodeErr == nil {
- timestamp = decodedTime.UnixNano()
+ if decoded, decodeErr := native.DecodeTimestamp(value); decodeErr == nil {
+ timestamp = decoded
}
case imVersionField:
version = convert.BytesToInt64(value)
@@ -401,7 +339,7 @@ func EnumerateGroupTarget(groupRoot string, intervalRule storage.IntervalRule, f
sidxDir := filepath.Join(segDir, directCopySidxDirName)
if info, statErr := os.Stat(sidxDir); statErr == nil && info.IsDir() {
- count, sidxErr := CountBlugeDocs(sidxDir)
+ count, sidxErr := CountSeriesIndexDocs(sidxDir)
if sidxErr != nil {
return nil, fmt.Errorf("seg %s sidx open: %w", e.Name(), sidxErr)
}
diff --git a/banyand/measure/query.go b/banyand/measure/query.go
index 3cd3e9ed8..a5f78eb6d 100644
--- a/banyand/measure/query.go
+++ b/banyand/measure/query.go
@@ -361,7 +361,8 @@ func (m *measure) searchSeriesList(ctx context.Context, series []*pbv1.Series, m
for i := range segments {
sd, sortedValues, err := segments[i].IndexDB().Search(ctx, series, storage.IndexSearchOpts{
- Query: mqo.Query,
+ Criteria: mqo.Criteria,
+ Fields: mqo.Fields,
Order: mqo.Order,
PreloadSize: preloadSize,
Projection: indexProjection,
@@ -547,11 +548,13 @@ func (m *measure) buildIndexQueryResult(ctx context.Context, mqo model.MeasureQu
}
var err error
opts := storage.IndexSearchOpts{
- Query: mqo.Query,
- Order: mqo.Order,
- PreloadSize: preloadSize,
- Projection: indexProjection,
- TimeRange: mqo.TimeRange,
+ Criteria: mqo.Criteria,
+ Fields: mqo.Fields,
+ IndexModeSubject: mqo.Name,
+ Order: mqo.Order,
+ PreloadSize: preloadSize,
+ Projection: indexProjection,
+ TimeRange: mqo.TimeRange,
}
seriesFilter := roaring.NewPostingList()
for i := range segments {
@@ -1119,6 +1122,12 @@ func (sr *segResult) remove(i int) {
}
sr.Timestamps = append(sr.Timestamps[:i], sr.Timestamps[i+1:]...)
sr.Versions = append(sr.Versions[:i], sr.Versions[i+1:]...)
+ if sr.TimestampSet != nil {
+ sr.TimestampSet = append(sr.TimestampSet[:i], sr.TimestampSet[i+1:]...)
+ }
+ if sr.VersionSet != nil {
+ sr.VersionSet = append(sr.VersionSet[:i], sr.VersionSet[i+1:]...)
+ }
if sr.sortedValues != nil {
sr.sortedValues = append(sr.sortedValues[:i], sr.sortedValues[i+1:]...)
}
diff --git a/banyand/measure/write_data_segmentref_test.go b/banyand/measure/write_data_segmentref_test.go
index bec688756..827bdd07f 100644
--- a/banyand/measure/write_data_segmentref_test.go
+++ b/banyand/measure/write_data_segmentref_test.go
@@ -36,6 +36,7 @@ import (
"github.com/apache/skywalking-banyandb/banyand/protector"
"github.com/apache/skywalking-banyandb/banyand/queue"
"github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
"github.com/apache/skywalking-banyandb/pkg/logger"
resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -198,7 +199,10 @@ func openTestTSDBForRefTest(t *testing.T, tmpPath string, shardNum uint32, openS
},
SegmentInterval: ir,
TTL: ir,
- Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
+ RootLeaseFactory: func(lock fs.File, root string) (storage.RootLease, error) {
+ return native.NewFileRootLease(lock, root)
+ },
+ Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
}
require.NoError(t, os.MkdirAll(opts.Location, storage.DirPerm))
ctx := common.SetPosition(
@@ -405,7 +409,10 @@ func openTestTSDBWithInterval(t *testing.T, tmpPath, groupName string, ir storag
TSTableCreator: newTSTable,
SegmentInterval: ir,
TTL: storage.IntervalRule{Unit: ir.Unit, Num: 60},
- Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
+ RootLeaseFactory: func(lock fs.File, root string) (storage.RootLease, error) {
+ return native.NewFileRootLease(lock, root)
+ },
+ Option: option{protector: protector.Nop{}, mergePolicy: newDefaultMergePolicyForTesting()},
}
require.NoError(t, os.MkdirAll(opts.Location, storage.DirPerm))
ctx := common.SetPosition(
diff --git a/banyand/metadata/schema/reader/e2e_native_schema_walk_test.go b/banyand/metadata/schema/reader/e2e_native_schema_walk_test.go
index 4f23302b0..1d3910fae 100644
--- a/banyand/metadata/schema/reader/e2e_native_schema_walk_test.go
+++ b/banyand/metadata/schema/reader/e2e_native_schema_walk_test.go
@@ -34,7 +34,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
)
// schemaCatalogWalker is the product seam issue #14011 names: the offline
@@ -52,18 +52,18 @@ var (
nidx01dCatalogSeam schemaCatalogWalker = WalkDocs
)
-// nativeSeamOfInverted is every identifier the schema reader may use from
-// pkg/index/inverted once the cutover has happened: the two read-only walks,
+// nativeReadOnlySeam is every identifier the schema reader may use from
+// pkg/index/native now that the cutover has happened: the two read-only walks,
// the one bounded selection they accept, the borrowed document they hand back,
// and the sentinels a caller classifies a read-only failure with.
//
-// A query builder, a query type, a store constructor or a segment type
+// A query builder, a query type, an owner constructor or a segment type
// appearing here is the schema reader building queries again, which is exactly
// what issue #14011's acceptance criteria forbid.
-var nativeSeamOfInverted = map[string]struct{}{
- "ErrCorruptIndex": {},
+var nativeReadOnlySeam = map[string]struct{}{
+ "ErrCorrupt": {},
"ErrInvalidSelection": {},
- "ErrNoCommittedIndex": {},
+ "ErrNoSnapshot": {},
"ReadOnlySelectDocuments": {},
"ReadOnlyWalkDocuments": {},
"StoredDocument": {},
@@ -75,7 +75,7 @@ var nativeSeamOfInverted = map[string]struct{}{
// catalog payloads with. Everything else it needs is BanyanDB's own code or the
// standard library.
//
-// The reader opens index directories through pkg/index/inverted's read-only
+// The reader opens index directories through pkg/index/native's read-only
// entry points, so it links no search library of its own; an import outside
// this set is the schema walk reaching past the native seam.
var allowedReaderExternalImports = map[string]struct{}{
@@ -180,9 +180,9 @@ func TestE2ESchemaWalkNativeMalformedMeasureIsTyped(t *testing.T) {
visited, err := walkNIDX01DCatalog(schema.KindMeasure)
tester.ErrorIs(err, ErrMalformedPropertyDocument,
"a malformed source must reach the caller as the schema reader's typed payload error")
- tester.NotErrorIs(err, inverted.ErrCorruptIndex,
+ tester.NotErrorIs(err, native.ErrCorrupt,
"a valid ICE container with malformed application data is not index corruption")
- tester.NotErrorIs(err, inverted.ErrNoCommittedIndex,
+ tester.NotErrorIs(err, native.ErrNoSnapshot,
"a damaged catalog is not an absent one; callers classify the two differently")
tester.Empty(visited, "a walk that fails must not also publish a partial catalog")
@@ -255,21 +255,25 @@ func TestE2ESchemaWalkNativeLeavesCorpusUnchanged(t *testing.T) {
//
// Requirement proved here:
//
-// R5 -- the schema reader links no search library of its own and reaches
-// pkg/index/inverted only through its read-only seam. Every third-party
-// import of the package is on an explicit allowlist, and every
-// identifier the package takes from pkg/index/inverted is one of the
-// read-only walks, the bounded selection they accept, the borrowed
-// document they yield, or a classification sentinel -- never a query
-// builder, a query type or a store constructor.
+// R5 -- the schema reader links no search library of its own, never imports
+// the retired pkg/index/inverted, and reaches pkg/index/native only
+// through its read-only seam. Every third-party import of the package
+// is on an explicit allowlist, and every identifier the package takes
+// from pkg/index/native is one of the read-only walks, the bounded
+// selection they accept, the borrowed document they yield, or a
+// classification sentinel -- never a query builder, a query type or an
+// owner constructor.
func TestE2ESchemaWalkNativeReachesNativeReader(t *testing.T) {
tester := require.New(t)
sources := 0
+ reachesWalk := false
for _, file := range readerPackageSources(t) {
sources++
imports := importQualifiersOf(t, file.syntax, file.path)
for _, importPath := range imports {
+ tester.NotEqual(invertedPackage, importPath,
+ "%s imports the retired %s; the schema walk must read through %s", file.path, invertedPackage, nativePackage)
if isStandardLibrary(importPath) || strings.HasPrefix(importPath, banyanDBModule) {
continue
}
@@ -277,18 +281,23 @@ func TestE2ESchemaWalkNativeReachesNativeReader(t *testing.T) {
tester.True(allowed, "%s imports %s outside the schema reader's dependency allowlist",
file.path, importPath)
}
- for _, used := range selectorsOnPackage(file.syntax, imports, invertedPackage) {
- _, allowed := nativeSeamOfInverted[used]
+ for _, used := range selectorsOnPackage(file.syntax, imports, nativePackage) {
+ _, allowed := nativeReadOnlySeam[used]
tester.True(allowed, "%s uses %s.%s, which is outside the read-only seam the schema walk may reach",
- file.path, filepath.Base(invertedPackage), used)
+ file.path, filepath.Base(nativePackage), used)
+ if used == "ReadOnlyWalkDocuments" {
+ reachesWalk = true
+ }
}
}
tester.NotZero(sources, "the schema reader package must have source files")
+ tester.True(reachesWalk, "the schema reader must walk catalog shards through %s.ReadOnlyWalkDocuments", nativePackage)
}
const (
banyanDBModule = "github.com/apache/skywalking-banyandb/"
invertedPackage = banyanDBModule + "pkg/index/inverted"
+ nativePackage = banyanDBModule + "pkg/index/native"
readerSourceGlob = "*.go"
)
diff --git a/banyand/metadata/schema/reader/reader.go b/banyand/metadata/schema/reader/reader.go
index de941d1cb..aa64e7354 100644
--- a/banyand/metadata/schema/reader/reader.go
+++ b/banyand/metadata/schema/reader/reader.go
@@ -35,7 +35,7 @@ import (
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema/property"
"github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
)
// Property-document field and directory names mirror the unexported layout
@@ -66,11 +66,11 @@ type Doc struct {
// for every document the requested kinds select. No kinds walks every live
// document.
func WalkShard(shardPath string, visit func(Doc) error, kinds ...schema.Kind) error {
- documentVisit := func(document inverted.StoredDocument) error {
+ documentVisit := func(document native.StoredDocument) error {
return decodeSchemaDocument(shardPath, document, visit)
}
if len(kinds) == 0 {
- if walkErr := inverted.ReadOnlyWalkDocuments(context.Background(), shardPath, documentVisit); walkErr != nil {
+ if walkErr := native.ReadOnlyWalkDocuments(context.Background(), shardPath, documentVisit); walkErr != nil {
return fmt.Errorf("walk schema docs in %s: %w", shardPath, walkErr)
}
return nil
@@ -79,14 +79,14 @@ func WalkShard(shardPath string, visit func(Doc) error, kinds ...schema.Kind) er
for kindIndex, kind := range kinds {
terms[kindIndex] = []byte(kind.String())
}
- selection := inverted.TermSelection{Field: index.IndexModeName, Terms: terms}
- if walkErr := inverted.ReadOnlySelectDocuments(context.Background(), shardPath, selection, documentVisit); walkErr != nil {
+ selection := native.TermSelection{Field: index.IndexModeName, Terms: terms}
+ if walkErr := native.ReadOnlySelectDocuments(context.Background(), shardPath, selection, documentVisit); walkErr != nil {
return fmt.Errorf("walk schema docs in %s: %w", shardPath, walkErr)
}
return nil
}
-func decodeSchemaDocument(shardPath string, document inverted.StoredDocument, visit func(Doc) error) error {
+func decodeSchemaDocument(shardPath string, document native.StoredDocument, visit func(Doc) error) error {
var sourceBytes []byte
var deleted bool
if visitErr := document.VisitStoredFields(func(field string, value []byte) bool {
diff --git a/banyand/metadata/schema/schemaserver/service.go b/banyand/metadata/schema/schemaserver/service.go
index ef72a48a2..bd2180e7f 100644
--- a/banyand/metadata/schema/schemaserver/service.go
+++ b/banyand/metadata/schema/schemaserver/service.go
@@ -252,7 +252,6 @@ func (s *server) PreRun(ctx context.Context) error {
Index: db.IndexConfig{
BatchWaitSec: batchWaitSeconds(s.flushTimeout),
WaitForPersistence: false,
- NativeWriter: true,
},
Snapshot: db.SnapshotConfig{
Location: s.snapshotDir,
diff --git a/banyand/property/db/db.go b/banyand/property/db/db.go
index 44c911916..c1cf28bd1 100644
--- a/banyand/property/db/db.go
+++ b/banyand/property/db/db.go
@@ -45,9 +45,7 @@ import (
obsservice "github.com/apache/skywalking-banyandb/banyand/observability/services"
"github.com/apache/skywalking-banyandb/banyand/property/gossip"
"github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index/inverted"
"github.com/apache/skywalking-banyandb/pkg/index/native"
- "github.com/apache/skywalking-banyandb/pkg/iter/sort"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/meter"
"github.com/apache/skywalking-banyandb/pkg/query"
@@ -86,9 +84,6 @@ type Database interface {
RegisterGossip(messenger gossip.Messenger)
// Close closes the database.
Close() error
- // SwitchIndexWriter drains the database and reopens every shard with the
- // requested writer while retaining the root ownership lock.
- SwitchIndexWriter(ctx context.Context, native bool) error
}
type groupShards struct {
@@ -117,10 +112,6 @@ type SnapshotConfig struct {
type IndexConfig struct {
BatchWaitSec int64
WaitForPersistence bool
- // NativeWriter selects the native owner and query/mutation path for every
- // shard this database opens, existing and newly created alike. False keeps
- // the legacy writer path for rollback and compatibility operation.
- NativeWriter bool
}
// Config holds the configuration for the property database.
@@ -142,21 +133,16 @@ type database struct {
nativeLease *native.FileRootLease
logger *logger.Logger
repairScheduler *repairScheduler
- snapshotFunc func(context.Context) (string, error)
groups sync.Map
repairBaseDir string
snapshotDir string
- repairBuildTreeCron string
location string
indexConfig IndexConfig
flushInterval time.Duration
expireDelete time.Duration
repairTreeSlotCount int
- quickBuildTreeTime time.Duration
mu sync.RWMutex
closed atomic.Bool
- transition atomic.Bool
- repairEnabled bool
}
// OpenDB opens a property database with the given configuration.
@@ -198,21 +184,15 @@ func OpenDB(ctx context.Context, cfg Config, omr observability.MetricsRegistry,
flushInterval: cfg.FlushInterval,
expireDelete: cfg.ExpireToDeleteDuration,
repairTreeSlotCount: cfg.Repair.TreeSlotCount,
- repairEnabled: cfg.Repair.Enabled,
- repairBuildTreeCron: cfg.Repair.BuildTreeCron,
- quickBuildTreeTime: cfg.Repair.QuickBuildTreeTime,
- snapshotFunc: cfg.Snapshot.Func,
repairBaseDir: cfg.Repair.Location,
snapshotDir: cfg.Snapshot.Location,
lfs: lfs,
indexConfig: cfg.Index,
lock: lock,
}
- if cfg.Index.NativeWriter {
- db.nativeLease, err = native.NewFileRootLease(lock, loc)
- if err != nil {
- return nil, err
- }
+ db.nativeLease, err = native.NewFileRootLease(lock, loc)
+ if err != nil {
+ return nil, err
}
// init repair scheduler
if cfg.Repair.Enabled {
@@ -294,9 +274,6 @@ func (db *database) load(ctx context.Context) error {
}
func (db *database) Update(ctx context.Context, shardID common.ShardID, id []byte, property *propertyv1.Property) error {
- if db.transition.Load() {
- return errors.New("database writer transition in progress")
- }
sd, err := db.loadShard(ctx, property.Metadata.Group, shardID)
if err != nil {
return err
@@ -317,9 +294,6 @@ func (db *database) Update(ctx context.Context, shardID common.ShardID, id []byt
}
func (db *database) Delete(ctx context.Context, docIDs [][]byte, delTime time.Time) error {
- if db.transition.Load() {
- return errors.New("database writer transition in progress")
- }
db.mu.RLock()
defer db.mu.RUnlock()
if db.closed.Load() {
@@ -347,9 +321,6 @@ func (db *database) Query(ctx context.Context, req *propertyv1.QueryRequest) ([]
if len(req.Groups) == 0 {
return nil, errors.New("property query requires at least one group")
}
- if db.transition.Load() {
- return nil, errors.New("database writer transition in progress")
- }
db.mu.RLock()
defer db.mu.RUnlock()
if db.closed.Load() {
@@ -363,32 +334,12 @@ func (db *database) Query(ctx context.Context, req *propertyv1.QueryRequest) ([]
if len(shards) == 0 {
return nil, nil
}
- if db.indexConfig.NativeWriter {
- if req.OrderBy == nil {
- var result []QueriedProperty
- for _, shardRef := range shards {
- hits, err := shardRef.searchNative(ctx, req, nil, int(req.Limit))
- if err != nil {
- return nil, err
- }
- for _, hit := range hits {
- result = append(result, hit)
- }
- }
- return result, nil
- }
- return db.queryNativeSorted(ctx, shards, req)
- }
- iq, err := inverted.BuildPropertyQuery(req, groupField, entityID)
- if err != nil {
- return nil, err
- }
if req.OrderBy == nil {
var result []QueriedProperty
for _, shardRef := range shards {
- hits, searchErr := shardRef.search(ctx, iq, nil, int(req.Limit))
- if searchErr != nil {
- return nil, searchErr
+ hits, err := shardRef.searchNative(ctx, req, nil, int(req.Limit))
+ if err != nil {
+ return nil, err
}
for _, hit := range hits {
result = append(result, hit)
@@ -396,26 +347,7 @@ func (db *database) Query(ctx context.Context, req *propertyv1.QueryRequest) ([]
}
return result, nil
}
- iters := make([]sort.Iterator[*queryProperty], 0, len(shards))
- for _, shardRef := range shards {
- hits, searchErr := shardRef.search(ctx, iq, req.OrderBy, int(req.Limit))
- if searchErr != nil {
- return nil, searchErr
- }
- if len(hits) > 0 {
- iters = append(iters, newQueryPropertyIterator(hits))
- }
- }
- if len(iters) == 0 {
- return nil, nil
- }
- mergeIter := sort.NewItemIter(iters, req.OrderBy.Sort == modelv1.Sort_SORT_DESC)
- defer mergeIter.Close()
- result := make([]QueriedProperty, 0, queryCapacityUint(req.Limit))
- for mergeIter.Next() {
- result = append(result, mergeIter.Val())
- }
- return result, nil
+ return db.queryNativeSorted(ctx, shards, req)
}
func (db *database) queryNativeSorted(ctx context.Context, shards []*shard, req *propertyv1.QueryRequest) ([]QueriedProperty, error) {
@@ -465,7 +397,7 @@ func (db *database) collectGroupShards(requestedGroups map[string]bool) []*shard
}
func (db *database) loadShard(ctx context.Context, group string, id common.ShardID) (*shard, error) {
- if db.closed.Load() || db.transition.Load() {
+ if db.closed.Load() {
return nil, errors.New("database is closed")
}
if s, ok := db.getShard(group, id); ok {
@@ -473,7 +405,7 @@ func (db *database) loadShard(ctx context.Context, group string, id common.Shard
}
db.mu.Lock()
defer db.mu.Unlock()
- if db.closed.Load() || db.transition.Load() {
+ if db.closed.Load() {
return nil, errors.New("database is closed")
}
return db.loadShardLocked(ctx, group, id)
@@ -538,9 +470,6 @@ func (db *database) getShard(group string, id common.ShardID) (*shard, bool) {
// Drop closes and removes all shards for the given group and deletes the group directory.
func (db *database) Drop(groupName string) (err error) {
- if db.transition.Load() {
- return errors.New("database writer transition in progress")
- }
db.mu.Lock()
defer db.mu.Unlock()
value, ok := db.groups.LoadAndDelete(groupName)
@@ -600,125 +529,6 @@ func (db *database) Close() error {
return err
}
-// SwitchIndexWriter performs a lease-preserving writer transition. Admission
-// is stopped and existing callbacks are drained before shards are closed; all
-// shards are then reopened in the requested mode while the root lock remains
-// held. A failed reopen fails closed and releases the lease only after every
-// opened resource has been cleaned up.
-func (db *database) SwitchIndexWriter(ctx context.Context, useNative bool) error {
- if db.closed.Load() {
- return errors.New("database is closed")
- }
- if !db.transition.CompareAndSwap(false, true) {
- return errors.New("database writer transition already in progress")
- }
- // The transition flag rejects new callbacks before the scheduler is
- // drained. Scheduler callbacks acquire db.mu themselves, so this must
- // happen before taking the database lock to avoid a wait cycle.
- if db.repairScheduler != nil {
- db.repairScheduler.close()
- db.repairScheduler = nil
- }
- db.mu.Lock()
- defer db.mu.Unlock()
-
- // Taking the lock here drains operations that were already admitted.
- if db.closed.Load() {
- db.transition.Store(false)
- return errors.New("database is closed")
- }
-
- closeShards := func() error {
- var closeErr error
- db.groups.Range(func(_, value any) bool {
- gs := value.(*groupShards)
- if shards := gs.shards.Load(); shards != nil {
- for _, shardRef := range *shards {
- multierr.AppendInto(&closeErr, shardRef.close())
- }
- }
- return true
- })
- db.groups.Range(func(key, _ any) bool {
- db.groups.Delete(key)
- return true
- })
- return closeErr
- }
- if closeErr := closeShards(); closeErr != nil {
- db.closed.Store(true)
- db.releaseNativeLease(&closeErr)
- db.transition.Store(false)
- return fmt.Errorf("close shards for writer transition: %w", closeErr)
- }
-
- if useNative && db.nativeLease == nil {
- lease, ownerErr := native.NewFileRootLease(db.lock, db.location)
- if ownerErr != nil {
- db.closed.Store(true)
- _ = db.lock.Close()
- db.transition.Store(false)
- return fmt.Errorf("acquire native writer ownership: %w", ownerErr)
- }
- db.nativeLease = lease
- }
- db.indexConfig.NativeWriter = useNative
-
- var loadErr error
- for _, groupDir := range lfs.ReadDir(db.location) {
- if !groupDir.IsDir() {
- continue
- }
- groupName := groupDir.Name()
- groupPath := filepath.Join(db.location, groupName)
- if walkErr := walkDir(groupPath, "shard-", func(suffix string) error {
- id, parseErr := strconv.Atoi(suffix)
- if parseErr != nil {
- return parseErr
- }
- _, shardErr := db.loadShardLocked(ctx, groupName, common.ShardID(id))
- return shardErr
- }); walkErr != nil {
- loadErr = walkErr
- break
- }
- }
- if loadErr != nil {
- if db.repairScheduler != nil {
- db.repairScheduler.close()
- db.repairScheduler = nil
- }
- cleanupErr := closeShards()
- db.closed.Store(true)
- db.releaseNativeLease(&cleanupErr)
- db.transition.Store(false)
- return fmt.Errorf("reopen shards for writer transition: %w", multierr.Append(loadErr, cleanupErr))
- }
- if db.repairEnabled {
- scheduler, schedulerErr := newRepairScheduler(db.logger, db.omr, db.metricsScope,
- db.repairBuildTreeCron, db.quickBuildTreeTime, db.repairTreeSlotCount, db, db.snapshotFunc)
- if schedulerErr != nil {
- cleanupErr := closeShards()
- db.closed.Store(true)
- db.releaseNativeLease(&cleanupErr)
- db.transition.Store(false)
- return fmt.Errorf("recreate repair scheduler for writer transition: %w", multierr.Append(schedulerErr, cleanupErr))
- }
- db.repairScheduler = scheduler
- db.groups.Range(func(_, value any) bool {
- gs := value.(*groupShards)
- if shards := gs.shards.Load(); shards != nil {
- for _, shardRef := range *shards {
- shardRef.repairState.scheduler = scheduler
- }
- }
- return true
- })
- }
- db.transition.Store(false)
- return nil
-}
-
func (db *database) collect() {
if db.closed.Load() {
return
@@ -737,8 +547,6 @@ func (db *database) collect() {
for _, s := range *sLst {
if s.nativeStore != nil {
s.nativeStore.collectMetrics()
- } else if s.store != nil {
- s.store.CollectMetrics()
}
}
return true
@@ -746,9 +554,6 @@ func (db *database) collect() {
}
func (db *database) Repair(ctx context.Context, id []byte, shardID uint64, property *propertyv1.Property, deleteTime int64) error {
- if db.transition.Load() {
- return errors.New("database writer transition in progress")
- }
s, err := db.loadShard(ctx, property.Metadata.Group, common.ShardID(shardID))
if err != nil {
return pkgerrors.WithMessagef(err, "failed to load shard %d", id)
@@ -766,9 +571,6 @@ func (db *database) Repair(ctx context.Context, id []byte, shardID uint64, prope
}
func (db *database) TakeSnapShot(ctx context.Context, sn string) *databasev1.Snapshot {
- if db.transition.Load() {
- return nil
- }
db.mu.RLock()
defer db.mu.RUnlock()
if db.closed.Load() {
@@ -797,12 +599,7 @@ func (db *database) TakeSnapShot(ctx context.Context, sn string) *databasev1.Sna
}
snpDir := path.Join(db.snapshotDir, sn, storage.DataDir, shardRef.group, filepath.Base(shardRef.location))
db.lfs.MkdirPanicIfExist(snpDir, storage.DirPerm)
- var snapshotErr error
- if shardRef.nativeStore != nil {
- snapshotErr = shardRef.nativeStore.takeFileSnapshot(snpDir)
- } else {
- snapshotErr = shardRef.store.TakeFileSnapshot(snpDir)
- }
+ snapshotErr := shardRef.nativeStore.takeFileSnapshot(snpDir)
if snapshotErr != nil {
db.logger.Error().Err(snapshotErr).Str("group", shardRef.group).
Str("shard", filepath.Base(shardRef.location)).Msg("fail to take shard snapshot")
diff --git a/banyand/property/db/db_lifetime_test.go b/banyand/property/db/db_lifetime_test.go
index 7f81058b2..aa7bb6794 100644
--- a/banyand/property/db/db_lifetime_test.go
+++ b/banyand/property/db/db_lifetime_test.go
@@ -17,6 +17,7 @@ package db
import (
"context"
+ "sync"
"testing"
"time"
@@ -26,69 +27,131 @@ import (
propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
"github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
+ "github.com/apache/skywalking-banyandb/pkg/index/native"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
)
-type blockingQuery struct{}
-
-func (blockingQuery) String() string { return "" }
+// This file proves database.Drop waits for an in-flight Delete/Update/
+// Repair/Query to finish using the shard it is about to close, and that the
+// shard's store refuses further use once closed -- against a REAL
+// *native.Owner-backed nativePropertyStore, not a fake implementing a
+// retired interface (the legacy index.SeriesStore the pre-NIDX-03 version of
+// this file's blockingStore implemented; shard.store/index.SeriesStore are
+// both gone).
+//
+// Two different seams make the in-flight call observably slow, because the
+// write and read paths differ in how (if at all) they can block at all:
+//
+// - Update (and Repair's own leading searchNative read -- see below)
+// block through a REAL, already-production seam: native.OwnerOptions.Persist
+// is a pluggable persistence function (its own doc comment: "Path enables
+// the built-in ICE snapshot publisher. An empty path keeps the owner in
+// memory-only mode unless Persist is supplied for a test seam"), and
+// nativePropertyStore.batch already waits on it when wait=true
+// (shard.go's production updateDocuments/batch call path is untouched;
+// only the test's own store construction supplies a blocking Persist).
+// - Query and Delete have no equivalent seam: native reads
+// (native.Owner.Acquire) are deliberately lock-free by design, so there
+// is no real hook that makes one observably slow. shard.go keeps one
+// minimal, unexported, documented hook for exactly this
+// (testBeforeNativeRead) -- see its doc comment on the shard struct.
+// Repair's own first step is a searchNative call (shard.go's repair()),
+// so the same read hook blocks it too; nothing about Repair's own
+// update step needs a seam.
+//
+// In every case, Drop not completing while the hook holds the shard "in
+// use" is still enforced purely by database.mu (RWMutex): Update/Delete/
+// Repair/Query all take db.mu.RLock() before touching the shard; Drop takes
+// db.mu.Lock(). The hooks only make that critical section long enough to
+// observe reliably; they do not implement the ordering guarantee itself.
-type blockingStore struct {
- index.SeriesStore
- entered chan struct{}
- updateEntered chan struct{}
- release chan struct{}
- closed chan struct{}
- usedAfterClose chan struct{}
+// blockGate is testBeforeNativeRead's test double: the first call closes
+// entered and then blocks until release is closed.
+type blockGate struct {
+ entered chan struct{}
+ release chan struct{}
+ once sync.Once
}
-func (s *blockingStore) BuildQuery([]index.SeriesMatcher, index.Query, *timestamp.TimeRange) (index.Query, error) {
- return blockingQuery{}, nil
+func newBlockGate() *blockGate {
+ return &blockGate{entered: make(chan struct{}), release: make(chan struct{})}
}
-func (s *blockingStore) Search(context.Context, []index.FieldKey, index.Query, int) ([]index.SeriesDocument, error) {
- close(s.entered)
- <-s.release
- select {
- case <-s.closed:
- close(s.usedAfterClose)
- default:
- }
- return nil, nil
+func (g *blockGate) hook() {
+ g.once.Do(func() { close(g.entered) })
+ <-g.release
}
-func (s *blockingStore) UpdateSeriesBatch(index.Batch) error {
- entered := s.updateEntered
- if entered == nil {
- entered = s.entered
- }
- close(entered)
- <-s.release
- select {
- case <-s.closed:
- close(s.usedAfterClose)
- default:
+// testRootLease is the minimal native.RootLease a memory-only (Path == "")
+// *native.Owner needs: NewOwner only calls Validate, never ValidatePath,
+// when Path is empty.
+type testRootLease struct{}
+
+func (testRootLease) Validate() error { return nil }
+
+// newRealMemoryShard builds a shard backed by a REAL, memory-only
+// (Path == "") *native.Owner wired through nativePropertyStore exactly as
+// newNativePropertyStore would, so a read (lookup/query) that runs after
+// testBeforeNativeRead's gate releases completes against a genuine owner
+// instead of a nil one.
+//
+// It supplies a trivial, immediately-completing Persist function rather than
+// leaving both Path and Persist unset. nativePropertyStore.batch always
+// attaches a PersistentCallback (independent of wait), and native.Owner
+// refuses any admission carrying one once Path == "" && Persist == nil,
+// because that combination never starts a persistence worker to invoke it
+// (see native.Owner's admitChecksLocked: queue == nil && hasCallback ->
+// ErrPersistenceConfiguration). Repair's success path (no older property
+// found) reaches updateDocuments/batch, so this shard needs a working
+// persistence queue even though it stays memory-only.
+func newRealMemoryShard(t *testing.T) *shard {
+ t.Helper()
+ owner, err := native.NewOwner(native.OwnerOptions{
+ Lease: testRootLease{},
+ Persist: func(context.Context, *native.ReadView) error { return nil },
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = owner.Close() })
+ return &shard{
+ nativeStore: &nativePropertyStore{owner: owner, wait: true},
+ id: 0,
+ repairState: &repair{},
+ l: logger.GetLogger("db_lifetime_test"),
}
- return nil
}
-func (s *blockingStore) Close() error {
- select {
- case <-s.closed:
- default:
- close(s.closed)
+// newBlockingWriteShard builds a shard with a REAL *native.Owner (memory-
+// only: Path is empty, Persist is the test's own blocking function) wired
+// through nativePropertyStore exactly as newNativePropertyStore would, so
+// updateDocuments's production code path (shard.go) is exercised unchanged.
+// release must be closed to let a persist -- and so the blocked Update --
+// complete.
+func newBlockingWriteShard(t *testing.T, entered, release chan struct{}) *shard {
+ t.Helper()
+ var once sync.Once
+ owner, err := native.NewOwner(native.OwnerOptions{
+ Lease: testRootLease{},
+ Persist: func(context.Context, *native.ReadView) error {
+ once.Do(func() { close(entered) })
+ <-release
+ return nil
+ },
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = owner.Close() })
+ return &shard{
+ nativeStore: &nativePropertyStore{owner: owner, wait: true},
+ id: 0,
+ repairState: &repair{},
+ l: logger.GetLogger("db_lifetime_test"),
}
- return nil
}
func TestDeleteAndDropCoordinateShardLifetime(t *testing.T) {
- store := &blockingStore{
- entered: make(chan struct{}),
- release: make(chan struct{}),
- closed: make(chan struct{}),
- usedAfterClose: make(chan struct{}),
- }
- shardList := []*shard{{store: store, id: 0}}
+ gate := newBlockGate()
+ sd := newRealMemoryShard(t)
+ sd.testBeforeNativeRead = gate.hook
+ shardList := []*shard{sd}
group := &groupShards{location: t.TempDir()}
group.shards.Store(&shardList)
var db database
@@ -98,7 +161,7 @@ func TestDeleteAndDropCoordinateShardLifetime(t *testing.T) {
go func() {
deleteDone <- db.Delete(context.Background(), [][]byte{[]byte("id")}, time.Now())
}()
- <-store.entered
+ <-gate.entered
dropDone := make(chan error, 1)
go func() { dropDone <- db.Drop("g") }()
select {
@@ -106,24 +169,15 @@ func TestDeleteAndDropCoordinateShardLifetime(t *testing.T) {
t.Fatal("Drop closed a shard while Delete was using it")
case <-time.After(10 * time.Millisecond):
}
- close(store.release)
+ close(gate.release)
require.NoError(t, <-deleteDone)
require.NoError(t, <-dropDone)
- select {
- case <-store.usedAfterClose:
- t.Fatal("Delete used a shard after Drop closed it")
- default:
- }
}
func TestUpdateAndDropCoordinateShardLifetime(t *testing.T) {
- store := &blockingStore{
- entered: make(chan struct{}),
- release: make(chan struct{}),
- closed: make(chan struct{}),
- usedAfterClose: make(chan struct{}),
- }
- shardList := []*shard{{store: store, id: 0, repairState: &repair{}}}
+ entered, release := make(chan struct{}), make(chan struct{})
+ sd := newBlockingWriteShard(t, entered, release)
+ shardList := []*shard{sd}
group := &groupShards{location: t.TempDir()}
group.shards.Store(&shardList)
var db database
@@ -136,7 +190,7 @@ func TestUpdateAndDropCoordinateShardLifetime(t *testing.T) {
Id: "id",
})
}()
- <-store.entered
+ <-entered
dropDone := make(chan error, 1)
go func() { dropDone <- db.Drop("g") }()
select {
@@ -144,25 +198,24 @@ func TestUpdateAndDropCoordinateShardLifetime(t *testing.T) {
t.Fatal("Drop closed a shard while Update was using it")
case <-time.After(10 * time.Millisecond):
}
- close(store.release)
+ close(release)
require.NoError(t, <-updateDone)
require.NoError(t, <-dropDone)
- select {
- case <-store.usedAfterClose:
- t.Fatal("Update used a shard after Drop closed it")
- default:
- }
+
+ // Use-after-close: the shard's own owner is now closed (Drop's s.close()
+ // called nativeStore.close(), which calls owner.Close()); a fresh
+ // attempt to use the SAME store must fail cleanly, not panic or corrupt
+ // state, confirming Drop's close really took effect rather than merely
+ // returning.
+ batchErr := sd.nativeStore.batch(context.Background(), index.Documents{{EntityValues: []byte("after-close")}}, nil)
+ require.ErrorIs(t, batchErr, native.ErrOwnerClosed, "the shard's store must refuse use after Drop closed it")
}
func TestRepairAndDropCoordinateShardLifetime(t *testing.T) {
- store := &blockingStore{
- entered: make(chan struct{}),
- updateEntered: make(chan struct{}),
- release: make(chan struct{}),
- closed: make(chan struct{}),
- usedAfterClose: make(chan struct{}),
- }
- shardList := []*shard{{store: store, id: 0, repairState: &repair{}}}
+ gate := newBlockGate()
+ sd := newRealMemoryShard(t)
+ sd.testBeforeNativeRead = gate.hook
+ shardList := []*shard{sd}
group := &groupShards{location: t.TempDir()}
group.shards.Store(&shardList)
var db database
@@ -175,7 +228,7 @@ func TestRepairAndDropCoordinateShardLifetime(t *testing.T) {
Id: "id",
}, 0)
}()
- <-store.entered
+ <-gate.entered
dropDone := make(chan error, 1)
go func() { dropDone <- db.Drop("g") }()
select {
@@ -183,12 +236,35 @@ func TestRepairAndDropCoordinateShardLifetime(t *testing.T) {
t.Fatal("Drop closed a shard while Repair was using it")
case <-time.After(10 * time.Millisecond):
}
- close(store.release)
+ close(gate.release)
require.NoError(t, <-repairDone)
require.NoError(t, <-dropDone)
+}
+
+func TestQueryAndDropCoordinateShardLifetime(t *testing.T) {
+ gate := newBlockGate()
+ sd := newRealMemoryShard(t)
+ sd.testBeforeNativeRead = gate.hook
+ shardList := []*shard{sd}
+ group := &groupShards{location: t.TempDir()}
+ group.shards.Store(&shardList)
+ var db database
+ db.lfs = fs.NewLocalFileSystem()
+ db.groups.Store("g", group)
+ queryDone := make(chan error, 1)
+ go func() {
+ _, err := db.Query(context.Background(), &propertyv1.QueryRequest{Groups: []string{"g"}})
+ queryDone <- err
+ }()
+ <-gate.entered
+ dropDone := make(chan error, 1)
+ go func() { dropDone <- db.Drop("g") }()
select {
- case <-store.usedAfterClose:
- t.Fatal("Repair used a shard after Drop closed it")
- default:
+ case <-dropDone:
+ t.Fatal("Drop closed a shard while Query was using it")
+ case <-time.After(10 * time.Millisecond):
}
+ close(gate.release)
+ require.NoError(t, <-queryDone)
+ require.NoError(t, <-dropDone)
}
diff --git a/banyand/property/db/e2e_native_legacy_acceptance_test.go b/banyand/property/db/e2e_native_legacy_acceptance_test.go
index 7ee8618d0..615ac8748 100644
--- a/banyand/property/db/e2e_native_legacy_acceptance_test.go
+++ b/banyand/property/db/e2e_native_legacy_acceptance_test.go
@@ -39,8 +39,7 @@ import (
)
// TestE2EPropertyNativeLegacyQueryMutationAcceptance exercises only public
-// Database operations in fresh native and legacy roots. It intentionally keeps
-// the two roots independent and compares the observable query/mutation trace.
+// Database operations in a fresh native root.
func TestE2EPropertyNativeLegacyQueryMutationAcceptance(t *testing.T) {
ctx := context.Background()
type result struct {
@@ -48,14 +47,14 @@ func TestE2EPropertyNativeLegacyQueryMutationAcceptance(t *testing.T) {
rows []string
count int
}
- run := func(nativeWriter bool) result {
+ run := func() result {
location, cleanup, err := test.NewSpace()
require.NoError(t, err)
defer cleanup()
cfg := Config{
Location: location, MetricsScopeName: "property_native_legacy_acceptance",
FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: nativeWriter, WaitForPersistence: true},
+ Index: IndexConfig{WaitForPersistence: true},
}
db, err := OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, err)
@@ -90,11 +89,8 @@ func TestE2EPropertyNativeLegacyQueryMutationAcceptance(t *testing.T) {
require.Equal(t, fingerprints, reopenedFingerprints)
return result{values: reopenedValues, rows: reopenedFingerprints, count: len(reopened)}
}
- legacy := run(false)
- native := run(true)
- sort.Slice(legacy.values, func(i, j int) bool { return legacy.values[i] < legacy.values[j] })
+ native := run()
sort.Slice(native.values, func(i, j int) bool { return native.values[i] < native.values[j] })
- require.Equal(t, legacy, native)
require.Equal(t, []int64{20, 30}, native.values)
require.Len(t, native.rows, 2)
require.Equal(t, 2, native.count)
@@ -106,14 +102,14 @@ func propertyRowFingerprint(row QueriedProperty) string {
hex.EncodeToString(row.Source())
}
-func TestE2EPropertyNativeRepairSnapshotAndWriterSwitch(t *testing.T) {
+func TestE2EPropertyNativeRepairSnapshotAndReopen(t *testing.T) {
ctx := context.Background()
location, cleanup, err := test.NewSpace()
require.NoError(t, err)
defer cleanup()
cfg := Config{
Location: location, MetricsScopeName: "property_native_repair_snapshot_acceptance",
- FlushInterval: time.Second, Index: IndexConfig{NativeWriter: true, WaitForPersistence: true},
+ FlushInterval: time.Second, Index: IndexConfig{WaitForPersistence: true},
Snapshot: SnapshotConfig{Location: filepath.Join(location, "snapshots")},
}
db, err := OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
@@ -135,7 +131,7 @@ func TestE2EPropertyNativeRepairSnapshotAndWriterSwitch(t *testing.T) {
require.NoError(t, copyAcceptanceTree(filepath.Join(cfg.Snapshot.Location, "acceptance", "data"), restoreLocation))
restored, restoreOpenErr := OpenDB(ctx, Config{
Location: restoreLocation, MetricsScopeName: "property_native_restore_acceptance", FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: true, WaitForPersistence: true},
+ Index: IndexConfig{WaitForPersistence: true},
Snapshot: SnapshotConfig{Location: filepath.Join(restoreLocation, "snapshots")},
}, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, restoreOpenErr)
@@ -147,20 +143,12 @@ func TestE2EPropertyNativeRepairSnapshotAndWriterSwitch(t *testing.T) {
require.Contains(t, propertyValues(t, queryDB(ctx, t, db, "repair-snapshot")), int64(11))
restored, restoreOpenErr = OpenDB(ctx, Config{
Location: restoreLocation, MetricsScopeName: "property_native_restore_acceptance_reopen", FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: true, WaitForPersistence: true},
+ Index: IndexConfig{WaitForPersistence: true},
Snapshot: SnapshotConfig{Location: filepath.Join(restoreLocation, "snapshots")},
}, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, restoreOpenErr)
require.Contains(t, propertyValues(t, queryDB(ctx, t, restored, "repair-snapshot")), int64(9))
require.NoError(t, restored.Close())
- require.NoError(t, db.SwitchIndexWriter(ctx, false))
- rows = queryDB(ctx, t, db, "repair-snapshot")
- require.Len(t, rows, 2)
- require.Contains(t, propertyValues(t, rows), int64(9))
- require.NoError(t, db.SwitchIndexWriter(ctx, true))
- rows = queryDB(ctx, t, db, "repair-snapshot")
- require.Len(t, rows, 2)
- require.Contains(t, propertyValues(t, rows), int64(9))
require.NoError(t, db.Close())
db, err = OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, err)
@@ -211,13 +199,13 @@ func propertyValues(t *testing.T, rows []QueriedProperty) []int64 {
func TestE2EPropertyNativeCriteriaAndSort(t *testing.T) {
ctx := context.Background()
- run := func(nativeWriter bool) (map[string][]string, []QueriedProperty, []QueriedProperty) {
+ run := func() (map[string][]string, []QueriedProperty, []QueriedProperty) {
location, cleanup, err := test.NewSpace()
require.NoError(t, err)
defer cleanup()
cfg := Config{
Location: location, MetricsScopeName: "property_native_criteria_acceptance", FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: nativeWriter, WaitForPersistence: true},
+ Index: IndexConfig{WaitForPersistence: true},
}
db, err := OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, err)
@@ -230,15 +218,12 @@ func TestE2EPropertyNativeCriteriaAndSort(t *testing.T) {
absent := generateProperty("f", 105, 0)
absent.Tags = nil
require.NoError(t, db.Update(ctx, 1, GetPropertyID(absent), absent))
- if nativeWriter {
- propertyDB := db.(*database)
- groupValue, loaded := propertyDB.groups.Load(testPropertyGroup)
- require.True(t, loaded)
- group := groupValue.(*groupShards)
- for _, shard := range *group.shards.Load() {
- require.NotNil(t, shard.nativeStore)
- require.Nil(t, shard.store)
- }
+ propertyDB := db.(*database)
+ groupValue, loaded := propertyDB.groups.Load(testPropertyGroup)
+ require.True(t, loaded)
+ group := groupValue.(*groupShards)
+ for _, shard := range *group.shards.Load() {
+ require.NotNil(t, shard.nativeStore)
}
queries := map[string]*modelv1.Criteria{
"eq": criteriaInt(modelv1.Condition_BINARY_OP_EQ, 30),
@@ -270,11 +255,7 @@ func TestE2EPropertyNativeCriteriaAndSort(t *testing.T) {
require.Len(t, queryOrderedLimit(ctx, t, db, modelv1.Sort_SORT_ASC, 1), 6)
return results, ascending, descending
}
- legacyResults, legacyAscending, legacyDescending := run(false)
- nativeResults, nativeAscending, nativeDescending := run(true)
- require.Equal(t, legacyResults, nativeResults)
- require.Equal(t, publicPropertyRows(legacyAscending), publicPropertyRows(nativeAscending))
- require.Equal(t, publicPropertyRows(legacyDescending), publicPropertyRows(nativeDescending))
+ nativeResults, nativeAscending, nativeDescending := run()
require.Equal(t, []string{"c"}, nativeResults["eq"])
require.Equal(t, []string{"a", "b", "d", "e", "f"}, nativeResults["ne"])
require.Equal(t, []string{"d", "e"}, nativeResults["gt"])
@@ -298,14 +279,6 @@ func TestE2EPropertyNativeCriteriaAndSort(t *testing.T) {
}
}
-func publicPropertyRows(rows []QueriedProperty) []string {
- result := make([]string, 0, len(rows))
- for _, row := range rows {
- result = append(result, propertyRowFingerprint(row)+":"+hex.EncodeToString(row.SortedValue()))
- }
- return result
-}
-
func criteriaInt(op modelv1.Condition_BinaryOp, value int64) *modelv1.Criteria {
return criteriaValues(op, &modelv1.TagValue{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: value}}})
}
diff --git a/banyand/property/db/e2e_native_writer_cutover_test.go b/banyand/property/db/e2e_native_writer_cutover_test.go
index 1dcab99a4..4813f0c5a 100644
--- a/banyand/property/db/e2e_native_writer_cutover_test.go
+++ b/banyand/property/db/e2e_native_writer_cutover_test.go
@@ -31,11 +31,11 @@ import (
"github.com/apache/skywalking-banyandb/pkg/test"
)
-// TestE2EPropertyNativeWriterCutoverAndRollback exercises the reversible
-// native-writer cutover through the real OpenDB -> newShard -> store path
-// issue #14075 (NIDX-02C) names as this leaf's single primary regression.
+// TestE2EPropertyNativeWriterCutoverAndRollback exercises OpenDB's lock
+// ownership and startup-failure cleanup through the real OpenDB -> newShard
+// -> store path issue #14075 (NIDX-02C) names.
//
-// Three requirements are proved, none of them assumed:
+// Two requirements are proved, none of them assumed:
//
// (1) lock-first ordering -- a second concurrent OpenDB against a Location
// that already holds a written shard must fail before opening (or
@@ -49,21 +49,12 @@ import (
// rather than panicking, and releases the lock: a subsequent clean
// OpenDB against the same Location must succeed rather than being
// blocked by a stale lock.
-// (3) rollback round-trips the issue's own observable sequence -- a shard
-// written Legacy, then Native, then reopened Native, then rolled back
-// to Legacy and written again, must read back the exact value trace
-// [legacy-v1, native-v2, native-v2, legacy-v3] the issue's primary
-// executable regression names. WaitForPersistence is set on every
-// config in this phase, so every Update call already blocks on its own
-// durable callback (shard.go's updateDocuments) before returning: no
-// durable callback is ever still in flight when Close runs, which is
-// what "durable callbacks drain before close" requires. This chains
-// two already-proven facts through the real production entrypoints:
-// TestNativeEncodedGenerationOpensInTheCompatibilityReader (native
-// encoder output opens in the unconditional-legacy NewStore) and
-// TestNIDX02BOutputMatchesTheNativeEncoder (the plugin's output is
-// byte-identical to that encoder's), so this is the first test to
-// combine both through OpenDB/newShard/NewStore.
+//
+// NIDX-03 removed the third requirement this test used to prove: the
+// legacy<->native writer round trip (SwitchIndexWriter no longer exists --
+// the binary opens every shard with the native store unconditionally, and
+// rollback is a file property exercised by the dedicated rollback harness,
+// not an in-process switch).
func TestE2EPropertyNativeWriterCutoverAndRollback(t *testing.T) {
tester := require.New(t)
@@ -78,7 +69,6 @@ func TestE2EPropertyNativeWriterCutoverAndRollback(t *testing.T) {
Location: lockDir,
MetricsScopeName: "property_native_cutover_lock_test",
FlushInterval: 3 * time.Second,
- Index: IndexConfig{NativeWriter: true},
}
db1, openErr := OpenDB(context.Background(), lockCfg, observability.BypassRegistry, fs.NewLocalFileSystem())
tester.NoError(openErr)
@@ -114,7 +104,6 @@ func TestE2EPropertyNativeWriterCutoverAndRollback(t *testing.T) {
TreeSlotCount: 1,
Enabled: true,
},
- Index: IndexConfig{NativeWriter: true},
}
// Seed a valid shard. The malformed directory below is deliberately
// encountered after this shard, so startup cleanup must close a partially
@@ -136,78 +125,4 @@ func TestE2EPropertyNativeWriterCutoverAndRollback(t *testing.T) {
db2, reopenErr := OpenDB(context.Background(), failureCfg, observability.BypassRegistry, fs.NewLocalFileSystem())
tester.NoError(reopenErr, "the failed OpenDB above must have released the lock, or this reopen fails")
tester.NoError(db2.Close())
-
- // (3) The issue's own primary observable: legacy-v1 -> native-v2 ->
- // (native reopen) native-v2 -> (legacy rollback) native-v2 -> legacy-v3.
- // WaitForPersistence: true on every config below matches production's
- // real property.service.go setting, so each Update already blocked on
- // its durable callback before returning -- Close always runs with
- // nothing left to drain.
- rollbackDir, rollbackCleanup, err := test.NewSpace()
- tester.NoError(err)
- defer rollbackCleanup()
-
- const compatibilityProbeID = "compatibility-probe"
- ctx := context.Background()
- readTag := func(db Database) int64 {
- results := queryDB(ctx, t, db, compatibilityProbeID)
- tester.Len(results, 1, "the compatibility-probe document must be readable")
- return unmarshalProperty(t, results[0].Source()).Tags[0].Value.GetInt().Value
- }
- openWithNative := func(nativeWriter bool) Database {
- cfg := Config{
- Location: rollbackDir,
- MetricsScopeName: "property_native_cutover_rollback_test",
- FlushInterval: 3 * time.Second,
- Index: IndexConfig{NativeWriter: nativeWriter, WaitForPersistence: true},
- }
- d, openErr := OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
- tester.NoError(openErr)
- return d
- }
-
- // The storage key a write's id names is fixed at the document's creation
- // (see banyand/property/listener.go:96, which forwards the caller's own
- // id rather than recomputing one per update): GetPropertyID folds in
- // ModRevision, so recomputing it per version would create a new document
- // each time instead of updating the one this test tracks.
- probeID := GetPropertyID(generateProperty(compatibilityProbeID, time.Now().UnixNano(), 1))
-
- // Seed one legacy-created shard: legacy-v1.
- legacyV1 := generateProperty(compatibilityProbeID, time.Now().UnixNano(), 1)
- legacyDB1 := openWithNative(false)
- tester.NoError(legacyDB1.Update(ctx, 0, probeID, legacyV1))
- tester.Equal(int64(1), readTag(legacyDB1), "legacy-v1 must be readable immediately after its durable write")
- tester.NoError(legacyDB1.Close())
-
- // Transition the same live database to Native while retaining its root
- // lease. Existing data remains readable without a bulk conversion.
- legacyDB := openWithNative(false)
- tester.Equal(int64(1), readTag(legacyDB), "the legacy-written value must be readable before cutover")
- tester.NoError(legacyDB.SwitchIndexWriter(ctx, true))
- tester.Equal(int64(1), readTag(legacyDB), "the native-selected transition must read the legacy value unchanged")
- nativeV2 := generateProperty(compatibilityProbeID, time.Now().UnixNano(), 2)
- tester.NoError(legacyDB.Update(ctx, 0, probeID, nativeV2))
- tester.Equal(int64(2), readTag(legacyDB), "native-v2 must be readable immediately after its durable write")
-
- // Reopen all shards in Native mode without releasing the root lease.
- tester.NoError(legacyDB.SwitchIndexWriter(ctx, true))
- tester.Equal(int64(2), readTag(legacyDB), "native-v2 must survive a native transition reopen")
-
- // Roll back in place. The lease remains held throughout close and reopen,
- // so another process cannot acquire a writer window between the modes.
- tester.NoError(legacyDB.SwitchIndexWriter(ctx, false))
- tester.Equal(int64(2), readTag(legacyDB), "the rollback transition must serve the native-written value with no data loss")
- legacyV3 := generateProperty(compatibilityProbeID, time.Now().UnixNano(), 3)
- tester.NoError(legacyDB.Update(ctx, 0, probeID, legacyV3))
- tester.Equal(int64(3), readTag(legacyDB), "legacy-v3 must be readable immediately after its durable write")
- tester.NoError(legacyDB.Close())
-
- // Close and reopen Legacy again: legacy-v3 persisted, completing the
- // [legacy-v1, native-v2, native-v2, legacy-v3] observable trace.
- legacyDB3 := openWithNative(false)
- defer func() {
- _ = legacyDB3.Close()
- }()
- tester.Equal(int64(3), readTag(legacyDB3), "legacy-v3 must survive a Legacy close and reopen")
}
diff --git a/banyand/property/db/native_expiry_test.go b/banyand/property/db/native_expiry_test.go
index 2434ba0b1..8b0650006 100644
--- a/banyand/property/db/native_expiry_test.go
+++ b/banyand/property/db/native_expiry_test.go
@@ -39,7 +39,7 @@ func TestNativeExpiredTombstoneCompactionKeepsPinnedView(t *testing.T) {
Location: location, MetricsScopeName: "property_native_expiry",
ExpireToDeleteDuration: time.Second,
Repair: RepairConfig{Location: filepath.Join(location, "repair")},
- Index: IndexConfig{NativeWriter: true, WaitForPersistence: true},
+ Index: IndexConfig{WaitForPersistence: true},
}
opened, err := OpenDB(ctx, cfg, observability.BypassRegistry, fs.NewLocalFileSystem())
require.NoError(t, err)
diff --git a/banyand/property/db/native_property.go b/banyand/property/db/native_property.go
index 53dedb7b0..fee00aeb0 100644
--- a/banyand/property/db/native_property.go
+++ b/banyand/property/db/native_property.go
@@ -27,6 +27,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/native"
+ "github.com/apache/skywalking-banyandb/pkg/index/native/criteria"
"github.com/apache/skywalking-banyandb/pkg/index/nativeanalysis"
"github.com/apache/skywalking-banyandb/pkg/query"
"github.com/apache/skywalking-banyandb/pkg/query/logical"
@@ -326,7 +327,7 @@ func (s *nativePropertyStore) matchRequest(ctx context.Context, view *native.Rea
return nil, err
}
if request.Criteria != nil {
- result, err = s.matchCriteria(ctx, view, result, request.Criteria)
+ result, err = criteria.Filter(ctx, view, result, request.Criteria, propertyFieldResolver{})
if err != nil {
return nil, err
}
@@ -334,177 +335,14 @@ func (s *nativePropertyStore) matchRequest(ctx context.Context, view *native.Rea
return result, nil
}
-type nativeHitKey struct {
- segment uint64
- document uint64
-}
-
-func (s *nativePropertyStore) matchCriteria(
- ctx context.Context, view *native.ReadView, universe []native.QueryHit, criteria *modelv1.Criteria,
-) ([]native.QueryHit, error) {
- if criteria == nil {
- return universe, nil
- }
- switch expression := criteria.GetExp().(type) {
- case *modelv1.Criteria_Condition:
- return s.matchCondition(ctx, view, universe, expression.Condition)
- case *modelv1.Criteria_Le:
- if expression.Le == nil || (expression.Le.Left == nil && expression.Le.Right == nil) {
- return nil, fmt.Errorf("logical expression has no operands: %w", logical.ErrInvalidLogicalExpression)
- }
- if expression.Le.Left == nil {
- return s.matchCriteria(ctx, view, universe, expression.Le.Right)
- }
- if expression.Le.Right == nil {
- return s.matchCriteria(ctx, view, universe, expression.Le.Left)
- }
- left, err := s.matchCriteria(ctx, view, universe, expression.Le.Left)
- if err != nil {
- return nil, err
- }
- right, err := s.matchCriteria(ctx, view, universe, expression.Le.Right)
- if err != nil {
- return nil, err
- }
- switch expression.Le.Op {
- case modelv1.LogicalExpression_LOGICAL_OP_OR:
- return unionHits(left, right), nil
- case modelv1.LogicalExpression_LOGICAL_OP_AND:
- return intersectHits(left, right), nil
- default:
- return nil, fmt.Errorf("unsupported logical operator %d: %w", expression.Le.Op, logical.ErrInvalidLogicalExpression)
- }
- default:
- return nil, logical.ErrInvalidCriteriaType
- }
-}
-
-func (s *nativePropertyStore) matchCondition(
- ctx context.Context, view *native.ReadView, universe []native.QueryHit, condition *modelv1.Condition,
-) ([]native.QueryHit, error) {
- if condition == nil || condition.Value == nil || condition.Value.Value == nil {
- return nil, logical.ErrUnsupportedConditionValue
- }
- if condition.Op == modelv1.Condition_BINARY_OP_MATCH {
- if _, ok := condition.Value.Value.(*modelv1.TagValue_Str); !ok {
- return nil, logical.ErrUnsupportedConditionValue
- }
- }
- if condition.Op == modelv1.Condition_BINARY_OP_IN || condition.Op == modelv1.Condition_BINARY_OP_NOT_IN {
- switch condition.Value.Value.(type) {
- case *modelv1.TagValue_StrArray, *modelv1.TagValue_IntArray:
- default:
- return nil, logical.ErrUnsupportedConditionValue
- }
- }
- expr, err := logical.ParseExpr(condition)
- if err != nil {
- return nil, err
- }
- field := propertyTagField(condition.Name)
- values := expr.Bytes()
- match := func(terms [][]byte, mode native.TermSetMode) ([]native.QueryHit, error) {
- return view.FilterTermsSet(ctx, universe, native.TermSetRequest{Field: field, Terms: terms, Mode: mode})
- }
- switch condition.Op {
- case modelv1.Condition_BINARY_OP_EQ:
- if len(values) != 1 {
- return nil, logical.ErrUnsupportedConditionOp
- }
- return match(values, native.MatchAnyTerm)
- case modelv1.Condition_BINARY_OP_IN:
- return match(values, native.MatchAnyTerm)
- case modelv1.Condition_BINARY_OP_HAVING:
- return match(values, native.MatchAllTerms)
- case modelv1.Condition_BINARY_OP_MATCH:
- if len(values) != 1 {
- return nil, logical.ErrUnsupportedConditionOp
- }
- terms, analyzeErr := nativeanalysis.Analyze(condition.MatchOption.GetAnalyzer(), values[0])
- if analyzeErr != nil {
- return nil, analyzeErr
- }
- analyzed := make([][]byte, 0, len(terms))
- for _, term := range terms {
- analyzed = append(analyzed, term.Value)
- }
- mode := native.MatchAnyTerm
- if condition.MatchOption.GetOperator() == modelv1.Condition_MatchOption_OPERATOR_AND {
- mode = native.MatchAllTerms
- }
- return match(analyzed, mode)
- case modelv1.Condition_BINARY_OP_GT, modelv1.Condition_BINARY_OP_GE,
- modelv1.Condition_BINARY_OP_LT, modelv1.Condition_BINARY_OP_LE:
- if len(values) != 1 {
- return nil, logical.ErrUnsupportedConditionOp
- }
- request := native.RangeRequest{Field: field, MaxTerms: ^uint64(0), Lower: nil, Upper: nil}
- switch condition.Op {
- case modelv1.Condition_BINARY_OP_GT, modelv1.Condition_BINARY_OP_GE:
- request.Lower, request.IncludesLower = values[0], condition.Op == modelv1.Condition_BINARY_OP_GE
- default:
- request.Upper, request.IncludesUpper = values[0], condition.Op == modelv1.Condition_BINARY_OP_LE
- }
- return view.FilterRange(ctx, universe, request)
- case modelv1.Condition_BINARY_OP_NE, modelv1.Condition_BINARY_OP_NOT_IN,
- modelv1.Condition_BINARY_OP_NOT_HAVING:
- if condition.Op == modelv1.Condition_BINARY_OP_NE && len(values) != 1 {
- return nil, logical.ErrUnsupportedConditionOp
- }
- var excluded []native.QueryHit
- mode := native.MatchAnyTerm
- if condition.Op == modelv1.Condition_BINARY_OP_NOT_HAVING {
- mode = native.MatchAllTerms
- }
- excluded, err = match(values, mode)
- if err != nil {
- return nil, err
- }
- return subtractHits(universe, excluded), nil
- default:
- return nil, logical.ErrUnsupportedConditionOp
- }
-}
-
-func intersectHits(left, right []native.QueryHit) []native.QueryHit {
- rightSet := make(map[nativeHitKey]native.QueryHit, len(right))
- for _, hit := range right {
- rightSet[nativeHitKey{hit.Segment, hit.DocumentNumber}] = hit
- }
- result := make([]native.QueryHit, 0, len(left))
- for _, hit := range left {
- if _, ok := rightSet[nativeHitKey{hit.Segment, hit.DocumentNumber}]; ok {
- result = append(result, hit)
- }
- }
- return result
-}
-
-func unionHits(left, right []native.QueryHit) []native.QueryHit {
- result := make([]native.QueryHit, 0, len(left)+len(right))
- seen := make(map[nativeHitKey]struct{}, len(left)+len(right))
- for _, hits := range [][]native.QueryHit{left, right} {
- for _, hit := range hits {
- key := nativeHitKey{hit.Segment, hit.DocumentNumber}
- if _, ok := seen[key]; !ok {
- seen[key] = struct{}{}
- result = append(result, hit)
- }
- }
- }
- return result
-}
+// propertyFieldResolver implements criteria.FieldResolver for Property: every
+// tag name resolves to its hashed engine field, matching propertyTagField's
+// use in the write path (shard.go) and in sort field resolution above. No
+// tag is ever rejected -- an unindexed tag simply has no postings, so EQ
+// finds nothing and NE finds everything -- and MATCH's analyzer always comes
+// from the request, never from a resolved schema value.
+type propertyFieldResolver struct{}
-func subtractHits(universe, excluded []native.QueryHit) []native.QueryHit {
- set := make(map[nativeHitKey]struct{}, len(excluded))
- for _, hit := range excluded {
- set[nativeHitKey{hit.Segment, hit.DocumentNumber}] = struct{}{}
- }
- result := make([]native.QueryHit, 0, len(universe))
- for _, hit := range universe {
- if _, ok := set[nativeHitKey{hit.Segment, hit.DocumentNumber}]; !ok {
- result = append(result, hit)
- }
- }
- return result
+func (propertyFieldResolver) Field(tagName string) (string, string, bool) {
+ return propertyTagField(tagName), "", true
}
diff --git a/banyand/property/db/native_property_match_test.go b/banyand/property/db/native_property_match_test.go
index 3b19b14ae..1bc1d641a 100644
--- a/banyand/property/db/native_property_match_test.go
+++ b/banyand/property/db/native_property_match_test.go
@@ -47,84 +47,67 @@ func TestNativePropertyMatchParity(t *testing.T) {
{name: "and", analyzer: "standard", operator: modelv1.Condition_MatchOption_OPERATOR_AND, want: []string{}},
{name: "or", analyzer: "standard", operator: modelv1.Condition_MatchOption_OPERATOR_OR, want: []string{"two"}},
}
- for _, nativeWriter := range []bool{false, true} {
- nativeWriter := nativeWriter
- t.Run("native="+boolString(nativeWriter), func(t *testing.T) {
- location, cleanup, err := test.NewSpace()
- require.NoError(t, err)
- defer cleanup()
- db, err := OpenDB(ctx, Config{
- Location: location, MetricsScopeName: "native_match_parity", FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: nativeWriter, WaitForPersistence: true},
- }, observability.BypassRegistry, fs.NewLocalFileSystem())
- require.NoError(t, err)
- defer db.Close()
- for revision, item := range []struct{ id, value string }{{"one", "red blue"}, {"two", "blue"}, {"three", "green"}} {
- p := &propertyv1.Property{
- Metadata: &commonv1.Metadata{Group: testPropertyGroup, Name: testPropertyName, ModRevision: int64(revision + 1)}, Id: item.id,
- Tags: []*modelv1.Tag{{Key: "tag1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: item.value}}}}},
- }
- require.NoError(t, db.Update(ctx, 0, GetPropertyID(p), p))
- }
- for _, tc := range cases {
- option := &modelv1.Condition_MatchOption{Analyzer: tc.analyzer, Operator: tc.operator}
- criteria := &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
- Name: "tag1", Op: modelv1.Condition_BINARY_OP_MATCH, MatchOption: option,
- Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "red blue"}}},
- }}}
- rows, queryErr := db.Query(ctx, &propertyv1.QueryRequest{Groups: []string{testPropertyGroup}, Criteria: criteria})
- require.NoError(t, queryErr, tc.name)
- got := make([]string, 0, len(rows))
- for _, row := range rows {
- got = append(got, unmarshalProperty(t, row.Source()).Id)
- }
- sort.Strings(got)
- require.Equal(t, tc.want, got, tc.name)
- }
- })
+ location, cleanup, err := test.NewSpace()
+ require.NoError(t, err)
+ defer cleanup()
+ db, err := OpenDB(ctx, Config{
+ Location: location, MetricsScopeName: "native_match_parity", FlushInterval: time.Second,
+ Index: IndexConfig{WaitForPersistence: true},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ require.NoError(t, err)
+ defer db.Close()
+ for revision, item := range []struct{ id, value string }{{"one", "red blue"}, {"two", "blue"}, {"three", "green"}} {
+ p := &propertyv1.Property{
+ Metadata: &commonv1.Metadata{Group: testPropertyGroup, Name: testPropertyName, ModRevision: int64(revision + 1)}, Id: item.id,
+ Tags: []*modelv1.Tag{{Key: "tag1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: item.value}}}}},
+ }
+ require.NoError(t, db.Update(ctx, 0, GetPropertyID(p), p))
}
-}
-
-func boolString(value bool) string {
- if value {
- return "true"
+ for _, tc := range cases {
+ option := &modelv1.Condition_MatchOption{Analyzer: tc.analyzer, Operator: tc.operator}
+ criteria := &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{
+ Name: "tag1", Op: modelv1.Condition_BINARY_OP_MATCH, MatchOption: option,
+ Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "red blue"}}},
+ }}}
+ rows, queryErr := db.Query(ctx, &propertyv1.QueryRequest{Groups: []string{testPropertyGroup}, Criteria: criteria})
+ require.NoError(t, queryErr, tc.name)
+ got := make([]string, 0, len(rows))
+ for _, row := range rows {
+ got = append(got, unmarshalProperty(t, row.Source()).Id)
+ }
+ sort.Strings(got)
+ require.Equal(t, tc.want, got, tc.name)
}
- return "false"
}
func TestNativePropertyEmptyStringConditions(t *testing.T) {
ctx := context.Background()
- for _, nativeWriter := range []bool{false, true} {
- nativeWriter := nativeWriter
- t.Run("native="+boolString(nativeWriter), func(t *testing.T) {
- location, cleanup, err := test.NewSpace()
- require.NoError(t, err)
- defer cleanup()
- db, err := OpenDB(ctx, Config{
- Location: location, MetricsScopeName: "native_empty_conditions", FlushInterval: time.Second,
- Index: IndexConfig{NativeWriter: nativeWriter, WaitForPersistence: true},
- }, observability.BypassRegistry, fs.NewLocalFileSystem())
- require.NoError(t, err)
- defer db.Close()
- p := &propertyv1.Property{
- Metadata: &commonv1.Metadata{Group: testPropertyGroup, Name: testPropertyName, ModRevision: 1}, Id: "empty",
- Tags: []*modelv1.Tag{{Key: "tag1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: ""}}}}},
- }
- require.NoError(t, db.Update(ctx, 0, GetPropertyID(p), p))
- for _, op := range []modelv1.Condition_BinaryOp{modelv1.Condition_BINARY_OP_EQ, modelv1.Condition_BINARY_OP_IN, modelv1.Condition_BINARY_OP_NOT_IN} {
- value := &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: ""}}}
- if op != modelv1.Condition_BINARY_OP_EQ {
- value = &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: []string{""}}}}
- }
- criteria := &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{Name: "tag1", Op: op, Value: value}}}
- rows, queryErr := db.Query(ctx, &propertyv1.QueryRequest{Groups: []string{testPropertyGroup}, Criteria: criteria})
- require.NoError(t, queryErr, op.String())
- if op == modelv1.Condition_BINARY_OP_NOT_IN {
- require.Empty(t, rows, op.String())
- } else {
- require.Len(t, rows, 1, op.String())
- }
- }
- })
+ location, cleanup, err := test.NewSpace()
+ require.NoError(t, err)
+ defer cleanup()
+ db, err := OpenDB(ctx, Config{
+ Location: location, MetricsScopeName: "native_empty_conditions", FlushInterval: time.Second,
+ Index: IndexConfig{WaitForPersistence: true},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ require.NoError(t, err)
+ defer db.Close()
+ p := &propertyv1.Property{
+ Metadata: &commonv1.Metadata{Group: testPropertyGroup, Name: testPropertyName, ModRevision: 1}, Id: "empty",
+ Tags: []*modelv1.Tag{{Key: "tag1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: ""}}}}},
+ }
+ require.NoError(t, db.Update(ctx, 0, GetPropertyID(p), p))
+ for _, op := range []modelv1.Condition_BinaryOp{modelv1.Condition_BINARY_OP_EQ, modelv1.Condition_BINARY_OP_IN, modelv1.Condition_BINARY_OP_NOT_IN} {
+ value := &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: ""}}}
+ if op != modelv1.Condition_BINARY_OP_EQ {
+ value = &modelv1.TagValue{Value: &modelv1.TagValue_StrArray{StrArray: &modelv1.StrArray{Value: []string{""}}}}
+ }
+ criteria := &modelv1.Criteria{Exp: &modelv1.Criteria_Condition{Condition: &modelv1.Condition{Name: "tag1", Op: op, Value: value}}}
+ rows, queryErr := db.Query(ctx, &propertyv1.QueryRequest{Groups: []string{testPropertyGroup}, Criteria: criteria})
+ require.NoError(t, queryErr, op.String())
+ if op == modelv1.Condition_BINARY_OP_NOT_IN {
+ require.Empty(t, rows, op.String())
+ } else {
+ require.Len(t, rows, 1, op.String())
+ }
}
}
diff --git a/banyand/property/db/nidx03_property_rollback_write_test.go b/banyand/property/db/nidx03_property_rollback_write_test.go
new file mode 100644
index 000000000..e1a211154
--- /dev/null
+++ b/banyand/property/db/nidx03_property_rollback_write_test.go
@@ -0,0 +1,129 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and limitations
+// under the License.
+
+package db
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+// NIDX-03 §15 Property file-rollback rerun, write side. #1390 recorded this
+// check once at 588cc602 (docs/design/0.12.0/native-inverted-index/
+// verification/property-native-cutover/README.md), against an unreleased
+// main commit; this reruns it against NIDX-03 §1's actual rollback target,
+// v0.11.1 (the last published release, which runs Property on the retired
+// bluge engine -- unlike 735e9ad2 or 588cc602, both already native), now
+// that the legacy writer switch itself is gone (IndexConfig.NativeWriter,
+// SwitchIndexWriter, shard.store -- banyand/property/db/{shard,db,
+// repair_gossip}.go). Property's on-disk format is unchanged by NIDX-03
+// (only its legacy branches were removed), so this is a narrower
+// re-confirmation, not a new claim.
+//
+// TestGenerateNIDX03PropertyRollbackCorpus writes properties (including an
+// update and a delete) with the current (native-only) code to
+// NIDX03_PROPERTY_ROLLBACK_DIR (default /mnt/d/tmp-gao-build/
+// nidx03-property-rollback; see nidx03PropertyRollbackDir), queries them
+// back, and dumps the query result as JSON beside the data. v0.11.1's own
+// copy of this package (staged by test/rollback/nidx03's previous-release
+// harness) opens the same directory with the exported OpenDB and Query --
+// unchanged in shape since before #1390 -- and is expected to match exactly.
+//
+// The property revision model this corpus exercises is NOT "later
+// ModRevision wins": GetPropertyID (group/name/id + "/" + ModRevision) makes
+// each revision its own physical document, so updating a property at a new
+// ModRevision adds a second live, independently queryable row rather than
+// superseding the first (see the two "rollback-update" rows -- ModRevision
+// 200 and 201 -- in the captured results; nothing collapses them). Only an
+// explicit Delete (a tombstone row) or two writes landing on the SAME
+// ModRevision (last write in that batch wins -- see
+// banyand/property/db/shard_test.go's "repair deleted version property with
+// same data" case) change which row(s) a query returns.
+func TestGenerateNIDX03PropertyRollbackCorpus(t *testing.T) {
+ if os.Getenv("NIDX03_ROLLBACK_WRITE") == "" {
+ t.Skip("one-off rollback-proof corpus generator; set NIDX03_ROLLBACK_WRITE=1 to (re)run it")
+ }
+ root := nidx03PropertyRollbackDir(t)
+ require.NoError(t, os.RemoveAll(root))
+ require.NoError(t, os.MkdirAll(root, 0o755))
+
+ ctx := context.Background()
+ database, err := OpenDB(ctx, Config{
+ Location: root, MetricsScopeName: "nidx03_property_rollback", FlushInterval: time.Second,
+ Index: IndexConfig{WaitForPersistence: true},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ require.NoError(t, err)
+
+ alive := generateProperty("rollback-alive", 100, 7)
+ require.NoError(t, database.Update(ctx, 0, GetPropertyID(alive), alive))
+ toUpdate := generateProperty("rollback-update", 200, 1)
+ require.NoError(t, database.Update(ctx, 0, GetPropertyID(toUpdate), toUpdate))
+ updated := generateProperty("rollback-update", 201, 2)
+ require.NoError(t, database.Update(ctx, 0, GetPropertyID(updated), updated))
+ toDelete := generateProperty("rollback-delete", 300, 1)
+ require.NoError(t, database.Update(ctx, 0, GetPropertyID(toDelete), toDelete))
+ require.NoError(t, database.Delete(ctx, [][]byte{GetPropertyID(toDelete)}, time.Now()))
+
+ rows, queryErr := database.Query(ctx, &propertyv1.QueryRequest{Groups: []string{testPropertyGroup}})
+ require.NoError(t, queryErr)
+ require.NoError(t, database.Close())
+
+ writeNIDX03PropertyResults(t, root, rows)
+}
+
+func nidx03PropertyRollbackDir(t *testing.T) string {
+ t.Helper()
+ if dir := os.Getenv("NIDX03_PROPERTY_ROLLBACK_DIR"); dir != "" {
+ return dir
+ }
+ return "/mnt/d/tmp-gao-build/nidx03-property-rollback"
+}
+
+// cross-version diff depends on (kept identical in the 735e9ad2-ported read
+// side); it is not a hot allocation path, so pointer-byte packing does not
+// matter here.
+//
+//nolint:govet // field order is the JSON key order this proof's byte-for-byte
+type nidx03PropertyRow struct {
+ ID string `json:"id"`
+ Timestamp int64 `json:"timestamp"`
+ DeleteTime int64 `json:"deleteTime"`
+ Source string `json:"source"`
+}
+
+func writeNIDX03PropertyResults(t *testing.T, root string, rows []QueriedProperty) {
+ t.Helper()
+ results := make([]nidx03PropertyRow, 0, len(rows))
+ for _, row := range rows {
+ results = append(results, nidx03PropertyRow{
+ ID: string(row.ID()), Timestamp: row.Timestamp(), DeleteTime: row.DeleteTime(), Source: string(row.Source()),
+ })
+ }
+ sort.Slice(results, func(i, j int) bool { return results[i].ID < results[j].ID })
+ data, err := json.MarshalIndent(results, "", " ")
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(root+"-results.json", data, 0o600))
+}
diff --git a/banyand/property/db/nidx03_property_upgrade_test.go b/banyand/property/db/nidx03_property_upgrade_test.go
new file mode 100644
index 000000000..f1c7c6050
--- /dev/null
+++ b/banyand/property/db/nidx03_property_upgrade_test.go
@@ -0,0 +1,191 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and limitations
+// under the License.
+
+package db
+
+import (
+ "context"
+ "fmt"
+ "path"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/encoding/protojson"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ "github.com/apache/skywalking-banyandb/banyand/observability"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index"
+ "github.com/apache/skywalking-banyandb/pkg/index/inverted"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+// TestNIDX03PropertyUpgradeFromLegacyBlugeLayout is R5's "lost coverage"
+// upgrade-direction companion to the rollback proof under test/rollback/
+// nidx03 (which covers the opposite direction -- v0.11.1 opening data THIS
+// code wrote). It always runs (no NIDX03_ROLLBACK gate): it never builds or
+// runs a previous release, it only writes a shard directory in the exact
+// shape the retired legacy writer (banyand/property/db/shard.go's removed
+// NativeWriter=false path, which called index.SeriesStore.UpdateSeriesBatch
+// with one bluge document keyed by EntityValues per property revision,
+// before NIDX-03 §15 removed the switch) would have left on disk, using
+// pkg/index/inverted.NewStore directly -- still in-tree for the NIDX-04
+// element-index migration tool -- then opens that SAME directory through
+// the current (native-only) OpenDB/Query path and asserts identical query
+// results against hand-written expectations.
+//
+// Field shape mirrors shard.go's buildUpdateDocument exactly (sourceField,
+// entityField, groupField, nameField, deletedField, plus one indexed field
+// per property tag), with one simplification: no shaValueField. That field
+// only feeds the anti-entropy repair/gossip protocol's content-hash
+// comparison (repair.buildShaValue), never Query -- a missing shaValue
+// changes what a repair round would detect as "differs," not what a query
+// returns, so this still exercises the real read path identically.
+func TestNIDX03PropertyUpgradeFromLegacyBlugeLayout(t *testing.T) {
+ tmpPath, cleanup := test.Space(require.New(t))
+ defer cleanup()
+
+ const group = "nidx03-upgrade-group"
+ const name = "nidx03-upgrade-name"
+ shardPath := path.Join(tmpPath, group, "shard-0")
+
+ alive := legacyTestProperty(group, name, "alive-id", 100, map[string]string{"status": "ok"})
+ toDelete := legacyTestProperty(group, name, "deleted-id", 200, map[string]string{"status": "gone"})
+ writeLegacyPropertyShard(t, shardPath, []legacyPropertyWrite{
+ {property: alive},
+ {property: toDelete},
+ {property: toDelete, deleteTime: time.Now().UnixNano()},
+ })
+
+ database, err := OpenDB(context.Background(), Config{
+ Location: tmpPath,
+ MetricsScopeName: "nidx03_property_upgrade_test",
+ FlushInterval: time.Second,
+ Index: IndexConfig{WaitForPersistence: true},
+ }, observability.BypassRegistry, fs.NewLocalFileSystem())
+ require.NoError(t, err, "the native-only OpenDB path must open a directory the retired legacy writer left behind")
+ defer func() { _ = database.Close() }()
+
+ rows, queryErr := database.Query(context.Background(), &propertyv1.QueryRequest{Groups: []string{group}})
+ require.NoError(t, queryErr)
+
+ byID := make(map[string]QueriedProperty, len(rows))
+ for _, row := range rows {
+ byID[string(row.ID())] = row
+ }
+
+ require.Len(t, rows, 2, "both revisions (one later soft-deleted) must be present; delete is a tombstone, not a removal")
+
+ aliveID := GetPropertyID(alive)
+ aliveRow, found := byID[string(aliveID)]
+ require.True(t, found, "the live property written by the legacy layout must be queryable through the native path")
+ require.Equal(t, int64(0), aliveRow.DeleteTime(), "the live property must not be marked deleted")
+ var decodedAlive propertyv1.Property
+ require.NoError(t, protojson.Unmarshal(aliveRow.Source(), &decodedAlive))
+ require.Equal(t, "alive-id", decodedAlive.GetId())
+ require.Equal(t, "ok", decodedAlive.GetTags()[0].GetValue().GetStr().GetValue())
+
+ deletedRowID := GetPropertyID(toDelete)
+ deletedRow, found := byID[string(deletedRowID)]
+ require.True(t, found, "the soft-deleted property must still be queryable (it is a tombstone, not absent)")
+ require.Greater(t, deletedRow.DeleteTime(), int64(0), "the soft-deleted property's deleteTime must be set")
+}
+
+type legacyPropertyWrite struct {
+ property *propertyv1.Property
+ deleteTime int64
+}
+
+func legacyTestProperty(group, name, id string, modRevision int64, tags map[string]string) *propertyv1.Property {
+ property := &propertyv1.Property{
+ Metadata: &commonv1.Metadata{Group: group, Name: name, ModRevision: modRevision},
+ Id: id,
+ }
+ for key, value := range tags {
+ property.Tags = append(property.Tags, &modelv1.Tag{
+ Key: key,
+ Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: value}}},
+ })
+ }
+ return property
+}
+
+// writeLegacyPropertyShard opens path with the retired bluge-backed store
+// (inverted.NewStore, still in-tree for the NIDX-04 element migration tool)
+// and writes each entry through UpdateSeriesBatch -- the same method and
+// document shape shard.go's buildUpdateDocument + updateDocuments used
+// before NIDX-03 removed the legacy writer switch -- then closes it so the
+// directory left behind is exactly what an upgrade finds.
+func writeLegacyPropertyShard(t *testing.T, shardPath string, writes []legacyPropertyWrite) {
+ t.Helper()
+ store, err := inverted.NewStore(inverted.StoreOpts{Path: shardPath, BatchWaitSec: 0})
+ require.NoError(t, err)
+ for _, w := range writes {
+ id := GetPropertyID(w.property)
+ doc, buildErr := legacyBuildUpdateDocument(id, w.property, w.deleteTime)
+ require.NoError(t, buildErr)
+ require.NoError(t, store.UpdateSeriesBatch(index.Batch{Documents: index.Documents{*doc}}))
+ }
+ require.NoError(t, store.Close())
+}
+
+// legacyBuildUpdateDocument mirrors shard.go's buildUpdateDocument (field
+// names, index/store flags) without the shaValueField -- see this file's
+// test doc comment for why that is a safe simplification for a query-path
+// proof.
+func legacyBuildUpdateDocument(id []byte, property *propertyv1.Property, deleteTime int64) (*index.Document, error) {
+ pj, err := protojson.Marshal(property)
+ if err != nil {
+ return nil, err
+ }
+ sourceField := index.NewBytesField(sourceFieldKey, pj)
+ sourceField.NoSort = true
+ sourceField.Store = true
+ entityField := index.NewBytesField(entityFieldKey, []byte(property.Id))
+ entityField.Index = true
+ groupField := index.NewBytesField(groupFieldKey, []byte(property.Metadata.Group))
+ groupField.Index = true
+ nameField := index.NewBytesField(nameFieldKey, []byte(property.Metadata.Name))
+ nameField.Index = true
+
+ doc := index.Document{
+ EntityValues: id,
+ Fields: []index.Field{entityField, groupField, nameField, sourceField},
+ Timestamp: property.Metadata.ModRevision,
+ }
+ for i, tag := range property.Tags {
+ tv, marshalErr := pbv1.MarshalTagValue(tag.Value)
+ if marshalErr != nil {
+ return nil, fmt.Errorf("tag %d: %w", i, marshalErr)
+ }
+ tagField := index.NewBytesField(index.FieldKey{IndexRuleID: uint32(convert.HashStr(tag.Key))}, tv)
+ tagField.Index = tag.Key != unindexedSourceTag
+ doc.Fields = append(doc.Fields, tagField)
+ }
+
+ if deleteTime > 0 {
+ deleteField := index.NewBytesField(deletedFieldKey, convert.Int64ToBytes(deleteTime))
+ deleteField.Store = true
+ deleteField.NoSort = true
+ doc.Fields = append(doc.Fields, deleteField)
+ }
+ return &doc, nil
+}
diff --git a/banyand/property/db/ownership_lifecycle_test.go b/banyand/property/db/ownership_lifecycle_test.go
deleted file mode 100644
index 07e14c82b..000000000
--- a/banyand/property/db/ownership_lifecycle_test.go
+++ /dev/null
@@ -1,163 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// Apache Software Foundation (ASF) licenses this file to you under the
-// Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License. You may obtain
-// a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package db
-
-import (
- "context"
- "os"
- "path/filepath"
- "runtime"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/banyand/observability"
- "github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/test"
-)
-
-// transitionBlockingStore makes an admitted update observable without relying
-// on a timer. Its persistent callback is delivered only after release, which
-// lets the transition test prove that SwitchIndexWriter waits for the update.
-type transitionBlockingStore struct {
- index.SeriesStore
- entered chan struct{}
- release chan struct{}
- closed chan struct{}
- enterOnce sync.Once
- closeOnce sync.Once
-}
-
-func (s *transitionBlockingStore) UpdateSeriesBatch(batch index.Batch) error {
- s.enterOnce.Do(func() { close(s.entered) })
- <-s.release
- if batch.PersistentCallback != nil {
- batch.PersistentCallback(nil)
- }
- return nil
-}
-
... 17657 lines suppressed ...