You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/09/16 07:14:18 UTC
(skywalking-banyandb) branch main updated: chore: remove the row-based query path from measure, stream and trace (#1326)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 403396797 chore: remove the row-based query path from measure, stream and trace (#1326)
403396797 is described below
commit 403396797a8829e338fc37a440cf6d1872d2fde9
Author: Bibek Jyoti Charah <bi...@gmail.com>
AuthorDate: Wed Sep 16 12:44:12 2026 +0530
chore: remove the row-based query path from measure, stream and trace (#1326)
* chore: default the e2e and soak harnesses to the vectorized path
---------
Co-authored-by: Gao Hongtao <ha...@gmail.com>
---
.github/workflows/e2e.yml | 24 +-
CHANGES.md | 1 +
api/data/codec.go | 17 +-
api/data/codec_test.go | 41 +-
api/data/frame_counters_test.go | 36 +-
api/data/stream_frame_test.go | 53 +-
api/data/trace_frame_test.go | 57 +-
api/proto/banyandb/measure/v1/query.proto | 2 +-
banyand/dquery/measure.go | 60 +-
banyand/dquery/measure_test.go | 123 +-
banyand/measure/block_batch_benchmark_test.go | 2 +-
banyand/measure/measure.go | 30 +-
banyand/measure/query_batch_top_test.go | 130 ++
banyand/measure/svc_data.go | 13 +-
banyand/measure/svc_liaison.go | 10 +-
banyand/measure/svc_standalone.go | 17 +-
banyand/query/processor.go | 189 +--
banyand/query/tracing_test.go | 47 +-
banyand/stream/benchmark_test.go | 203 ---
banyand/stream/block.go | 98 --
banyand/stream/query.go | 324 +----
banyand/stream/query_by_idx.go | 366 -----
banyand/stream/query_by_idx_test.go | 325 -----
banyand/stream/query_by_ts.go | 304 ----
banyand/stream/query_by_ts_test.go | 266 ----
banyand/stream/query_capacity.go | 26 -
banyand/stream/query_capacity_test.go | 77 -
banyand/stream/query_vectorized_parity_test.go | 645 ++++-----
banyand/stream/query_vectorized_test.go | 208 ---
banyand/stream/stream.go | 3 +-
banyand/stream/svc_liaison.go | 3 +-
banyand/stream/svc_standalone.go | 7 +-
banyand/stream/testdata/parity_boundary.yaml | 330 +++++
.../stream/testdata/parity_large_single_block.yaml | 1517 ++++++++++++++++++++
banyand/stream/testdata/parity_topn_desc.yaml | 27 +
.../testdata/parity_topn_multiseries_asc.yaml | 47 +
banyand/stream/tracing.go | 35 -
banyand/stream/vectorized_flag_test.go | 57 +
banyand/trace/query.go | 55 +-
banyand/trace/query_capacity_test.go | 45 -
banyand/trace/query_vectorized_test.go | 4 +-
banyand/trace/streaming_pipeline.go | 829 +----------
banyand/trace/streaming_pipeline_test.go | 864 -----------
banyand/trace/svc_liaison.go | 8 +-
banyand/trace/svc_standalone.go | 11 +-
banyand/trace/trace.go | 11 +-
banyand/trace/vectorized_bench_test.go | 79 -
banyand/trace/vectorized_flag_test.go | 41 +
banyand/trace/vectorized_parity_test.go | 124 +-
docs/api-reference.md | 2 +-
docs/operation/observability/tracing.md | 4 +-
docs/operation/troubleshooting/error-checklist.md | 1 -
.../measure-vec-flag-off-rollback.md | 71 -
docs/operation/troubleshooting/query.md | 2 +-
docs/operation/upgrade.md | 46 +-
docs/soak/g5d-phase-d-summary.md | 7 +
docs/soak/g5d-runbook.md | 7 +
docs/soak/g9f-runbook.md | 7 +
docs/soak/trace-soak-runbook.md | 80 +-
pkg/panicdiag/lintrawgo/baseline.txt | 1 -
pkg/query/executor/interface.go | 24 +-
pkg/query/logical/measure/measure_analyzer.go | 246 ----
pkg/query/logical/measure/measure_plan.go | 119 --
.../logical/measure/measure_plan_aggregation.go | 369 -----
.../logical/measure/measure_plan_distributed.go | 644 ---------
pkg/query/logical/measure/measure_plan_groupby.go | 299 ----
.../measure/measure_plan_indexscan_local.go | 519 -------
.../measure/measure_plan_indexscan_local_test.go | 163 ---
pkg/query/logical/measure/measure_plan_merge.go | 176 ---
pkg/query/logical/measure/measure_plan_top.go | 168 ---
pkg/query/logical/measure/measure_top.go | 174 ---
pkg/query/logical/measure/measure_top_test.go | 130 --
pkg/query/logical/measure/schema.go | 37 +-
pkg/query/logical/measure/sortable_data_points.go | 49 +
pkg/query/logical/measure/sorted_miterator.go | 195 +++
...istributed_test.go => sorted_miterator_test.go} | 131 --
pkg/query/logical/stream/stream_analyzer.go | 43 +-
.../logical/stream/stream_plan_indexscan_local.go | 131 +-
.../stream_plan_indexscan_local_vectorized.go | 126 +-
.../stream_plan_indexscan_local_vectorized_test.go | 77 +-
pkg/query/logical/stream/stream_plan_merge.go | 36 +-
pkg/query/logical/stream/stream_plan_tag_filter.go | 38 +-
.../stream/stream_plan_topn_pushdown_test.go | 4 -
pkg/query/model/model.go | 198 ---
pkg/query/model/model_test.go | 1245 ----------------
pkg/query/vectorized/measure/bench_test.go | 2 +-
pkg/query/vectorized/measure/config.go | 8 +-
pkg/query/vectorized/measure/config_test.go | 5 +-
pkg/query/vectorized/measure/diff_test.go | 8 +-
pkg/query/vectorized/measure/plan/dispatch.go | 130 +-
pkg/query/vectorized/measure/plan/dispatch_test.go | 179 +--
.../vectorized/measure/plan/distributed_test.go | 28 +-
pkg/query/vectorized/measure/plan/executor_test.go | 2 +-
pkg/query/vectorized/measure/plan/plan.go | 6 +-
pkg/query/vectorized/stream/config.go | 26 +-
pkg/query/vectorized/stream/config_test.go | 19 +-
pkg/query/vectorized/stream/distinct.go | 7 +-
pkg/query/vectorized/trace/config.go | 18 +-
pkg/query/vectorized/trace/config_test.go | 4 -
scripts/soak-vectorized.sh | 50 +-
test/e2e-v2/cases/cluster/docker-compose.yml | 4 +-
.../e2e-v2/cases/event/banyandb/docker-compose.yml | 2 +-
.../lifecycle/data-generator/docker-compose.yml | 2 +-
.../lifecycle/dns-discovery/banyandb-cluster.yaml | 14 +-
test/e2e-v2/cases/lifecycle/dns-discovery/e2e.yaml | 6 -
.../lifecycle/file-discovery/docker-compose.yml | 8 +-
test/e2e-v2/cases/log/banyandb/docker-compose.yml | 2 +-
.../profiling/trace/banyandb/docker-compose.yml | 2 +-
.../istio/banyandb/banyandb-deployment-vec.yaml | 68 -
.../istio/banyandb/banyandb-deployment.yaml | 2 -
.../rover/process/istio/banyandb/e2e-banyandb.yaml | 2 +-
.../cases/storage/banyandb/docker-compose.yml | 2 +-
.../cases/zipkin/banyandb/docker-compose.yml | 2 +-
.../distributed/query/vectorized_stream_test.go | 29 +-
.../distributed/query/vectorized_test.go | 43 +-
.../distributed/query/vectorized_trace_test.go | 20 +-
test/integration/distributed/querybench/README.md | 12 +-
.../distributed/querybench/benchmark_test.go | 115 +-
test/integration/distributed/querybench/config.go | 15 +-
.../distributed/querybench/config_test.go | 42 +-
.../distributed/querybench/orchestrate.sh | 2 +-
test/integration/distributed/querybench/report.go | 62 +-
.../distributed/querybench/report_test.go | 18 +-
.../distributed/querybench/run-docker.sh | 2 +-
.../distributed/querybench/soak_test.go | 8 +-
.../standalone/query/vectorized_stream_test.go | 18 +-
.../standalone/query/vectorized_test.go | 30 +-
.../standalone/query/vectorized_trace_test.go | 17 +-
test/soak/docker-compose.soak-distributed.yaml | 15 +-
test/soak/docker-compose.soak.yaml | 7 +-
130 files changed, 3587 insertions(+), 10836 deletions(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 46cadf1f4..602869847 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -55,23 +55,12 @@ jobs:
tag: ${{ github.sha }}
StoragePlugins:
- name: ${{ matrix.test.name }}${{ matrix.engine.suffix }}
+ name: ${{ matrix.test.name }}
runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }}
timeout-minutes: 90
needs: [DockerImage]
strategy:
fail-fast: false
- # Two-dimensional matrix: (suite × engine). The `engine` axis pivots
- # every suite into a (row, vec) pair without literal duplication. The
- # row variant keeps the historical CI label (empty suffix); the vec
- # variant appends " Vec" so existing dashboards/regex over job names
- # keep matching.
- #
- # BANYANDB_DEPLOYMENT_FILE is consumed only by the Rover
- # e2e-banyandb.yaml. It is exported by the "Set env var" step below
- # (conditional on matrix.test.name=='Rover...' && engine.suffix==' Vec')
- # rather than via matrix.engine.env so it does not bloat the
- # skywalking-infra-e2e log directory name past Linux NAME_MAX=255.
matrix:
test:
- name: Storage
@@ -97,11 +86,6 @@ jobs:
config: test/e2e-v2/cases/lifecycle/file-discovery/e2e.yaml
- name: Lifecycle DNS Discovery
config: test/e2e-v2/cases/lifecycle/dns-discovery/e2e.yaml
- engine:
- - suffix: ""
- - suffix: " Vec"
- env: |
- BANYANDB_MEASURE_VECTORIZED_FLAGS=--measure-vectorized-enabled=true
env:
TAG: ${{ github.sha }}
steps:
@@ -132,7 +116,6 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set env var
run: |
- echo "${{ matrix.engine.env }}" >> $GITHUB_ENV
# Suite-specific env is set here rather than via matrix.test.env
# to keep the skywalking-infra-e2e log directory name (built from
# the full matrix object) under Linux NAME_MAX=255 — particularly
@@ -141,9 +124,6 @@ jobs:
"Rover eBPF"*)
echo "ISTIO_VERSION=1.28.0" >> $GITHUB_ENV
echo "KUBERNETES_VERSION=28" >> $GITHUB_ENV
- if [[ "${{ matrix.engine.suffix }}" == " Vec" ]]; then
- echo "BANYANDB_DEPLOYMENT_FILE=test/e2e-v2/cases/rover/process/istio/banyandb/banyandb-deployment-vec.yaml" >> $GITHUB_ENV
- fi
;;
esac
- name: Generate data for lifecycle
@@ -172,7 +152,7 @@ jobs:
if: ${{ (failure() && startsWith(matrix.test.name, 'Lifecycle')) }}
name: Upload generated data
with:
- name: lifecycle-data-${{ matrix.test.name }}${{ matrix.engine.suffix }}
+ name: lifecycle-data-${{ matrix.test.name }}
path: "test/e2e-v2/cases/lifecycle/data-generator/tmp"
Storage:
diff --git a/CHANGES.md b/CHANGES.md
index 1c23cc0a1..b3d3cc6df 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -7,6 +7,7 @@ Release Notes.
### Features
- Add logging related flags to the FODC proxy and agent. Every BanyanDB binary now shares the same logging flags and the matching `BYDB_LOGGING_*` environment variables.
+- [Breaking Change] Remove the row-based query execution path from the Stream, Measure and Trace query engines. Measure TopN pre-aggregation is unchanged and still executes row-based. The `--stream-vectorized-enabled`, `--trace-vectorized-enabled` and `--measure-vectorized-enabled` flags stay registered but no longer select an engine; `=false` now fails fast at startup on standalone, data and liaison nodes. A query shape the vectorized engine cannot plan now returns an error instead of f [...]
- Push the criteria tag filter ahead of the vectorized stream merge, so a filtered index-order query bounds its merge at limit+offset. For such a query, where several rows share an ElementID, the criteria is now evaluated first and the element is represented by its first matching row in the requested sort order. Filtered timestamp-order queries are unchanged.
### Bug Fixes
diff --git a/api/data/codec.go b/api/data/codec.go
index 984bcd550..31c46d61d 100644
--- a/api/data/codec.go
+++ b/api/data/codec.go
@@ -129,22 +129,21 @@ func (c *RawFrameCodec) Unmarshal(body []byte) (any, error) {
}
// measureWireModeRaw is the per-process wire mode for TopicInternalMeasureQuery.
-// It is set once at measure-service startup from --measure-vectorized-enabled
-// (a per-process CLI bool with zero cluster propagation — G9f spec Principle 2).
-// Default false (flag-off / proto) so any process that never sets it keeps the
-// pre-G9f.0 proto behavior.
+// It is set once at measure-service startup. Default false (proto) so any
+// process that never sets it — notably a traced query, which forces the proto
+// egress — keeps the pre-G9f.0 proto behavior.
var measureWireModeRaw atomic.Bool
// SetMeasureWireModeRaw publishes the per-process wire mode for
-// TopicInternalMeasureQuery. raw==true selects RawFrameCodec (flag-on,
-// vec columnar frame body); raw==false selects ProtoCodec (flag-off, proto
-// body). Called at measure-service startup after flags are parsed.
+// TopicInternalMeasureQuery. raw==true selects RawFrameCodec (vec columnar
+// frame body); raw==false selects ProtoCodec. Called at measure-service
+// startup after flags are parsed.
func SetMeasureWireModeRaw(raw bool) {
measureWireModeRaw.Store(raw)
}
-// MeasureWireModeRaw reports whether this process is flag-on (raw vec frame
-// body) for TopicInternalMeasureQuery.
+// MeasureWireModeRaw reports whether this process uses the raw vec frame
+// body for TopicInternalMeasureQuery.
func MeasureWireModeRaw() bool {
return measureWireModeRaw.Load()
}
diff --git a/api/data/codec_test.go b/api/data/codec_test.go
index ef6a0462a..0afaf6fb2 100644
--- a/api/data/codec_test.go
+++ b/api/data/codec_test.go
@@ -146,12 +146,11 @@ func TestTopicResponseMap_ProtoCodec_RoundTripsByteIdentical(t *testing.T) {
}
}
-// TestMeasureQueryResponseCodec_DispatchesOnWireMode covers the G9f.0
-// topic-AND-process-wire-mode guard: TopicInternalMeasureQuery is served by a
-// single static supplier in the map but the codec selected per-encode/decode
-// must depend on the per-process wire-mode flag set via SetMeasureWireModeRaw
-// (flag-on → RawFrameCodec; flag-off → ProtoCodec).
-func TestMeasureQueryResponseCodec_DispatchesOnWireMode(t *testing.T) {
+// TestMeasureQueryResponseCodec_RawPassthrough covers the raw-frame half of the
+// TopicInternalMeasureQuery codec: a magic-prefixed []byte body is passed through
+// unchanged in both directions, and a body carrying the wrong magic falls back to
+// the proto envelope decoder and fails loud rather than being silently accepted.
+func TestMeasureQueryResponseCodec_RawPassthrough(t *testing.T) {
codec, ok := TopicResponseMap[TopicInternalMeasureQuery]
if !ok {
t.Fatal("TopicInternalMeasureQuery is not registered in TopicResponseMap")
@@ -162,44 +161,28 @@ func TestMeasureQueryResponseCodec_DispatchesOnWireMode(t *testing.T) {
prev := MeasureWireModeRaw()
t.Cleanup(func() { SetMeasureWireModeRaw(prev) })
- // Flag OFF: proto behavior, byte-identical to pre-G9f.0 path for the
- // real measurev1.InternalQueryResponse{} body.
- SetMeasureWireModeRaw(false)
- msg := &measurev1.InternalQueryResponse{}
- body, err := codec.Marshal(msg)
- if err != nil {
- t.Fatalf("flag-off Marshal failed: %v", err)
- }
- decoded, err := codec.Unmarshal(body)
- if err != nil {
- t.Fatalf("flag-off Unmarshal failed: %v", err)
- }
- if _, ok := decoded.(*measurev1.InternalQueryResponse); !ok {
- t.Fatalf("flag-off Unmarshal returned %T; want *measurev1.InternalQueryResponse", decoded)
- }
-
- // Flag ON: raw passthrough; bytes in == bytes out, 0x00-leading.
SetMeasureWireModeRaw(true)
+ // Raw passthrough: bytes in == bytes out, 0x00-leading.
rawBody := []byte{RawFrameMagicLeadingByte, 0x01, 0xAA, 0xBB}
encoded, err := codec.Marshal(rawBody)
if err != nil {
- t.Fatalf("flag-on Marshal failed: %v", err)
+ t.Fatalf("raw Marshal failed: %v", err)
}
if !bytes.Equal(encoded, rawBody) {
- t.Fatalf("flag-on Marshal is not passthrough: got %x want %x", encoded, rawBody)
+ t.Fatalf("raw Marshal is not passthrough: got %x want %x", encoded, rawBody)
}
out, err := codec.Unmarshal(rawBody)
if err != nil {
- t.Fatalf("flag-on Unmarshal of 0x00-leading body failed: %v", err)
+ t.Fatalf("raw Unmarshal of 0x00-leading body failed: %v", err)
}
outBytes, _ := out.([]byte)
if !bytes.Equal(outBytes, rawBody) {
- t.Fatalf("flag-on Unmarshal is not passthrough: got %x want %x", outBytes, rawBody)
+ t.Fatalf("raw Unmarshal is not passthrough: got %x want %x", outBytes, rawBody)
}
- // Flag ON + body with the wrong magic now routes to the proto envelope decoder.
+ // A body with the wrong magic routes to the proto envelope decoder.
if _, err := codec.Unmarshal([]byte{0xff, 0x02}); err == nil {
- t.Fatal("flag-on Unmarshal of invalid proto envelope returned nil error; want fail-loud")
+ t.Fatal("Unmarshal of invalid proto envelope returned nil error; want fail-loud")
}
}
diff --git a/api/data/frame_counters_test.go b/api/data/frame_counters_test.go
index fc41e9c6e..e8e8d997e 100644
--- a/api/data/frame_counters_test.go
+++ b/api/data/frame_counters_test.go
@@ -30,7 +30,7 @@ import (
// The counters exist to let a soak or an operator assert that the native columnar
// frame is genuinely carrying traffic. Their whole value is that they are zero when
// it is not, so both directions are pinned here: a frame body must move them, and a
-// flag-off process must leave them alone.
+// proto response must leave them alone.
//
// Encode and decode are counted at different layers because the wire path is not
// symmetric. The liaison decodes through ResponseCodec.Unmarshal, so the decode
@@ -44,23 +44,12 @@ func TestStreamFrameCounters_CountRawTrafficOnly(t *testing.T) {
c := newStreamCodec()
frame := []byte{RawFrameMagicLeadingByte, 0x01, 0x02, 0x03}
- // Flag OFF: a proto response must not be counted as frame traffic. This is the
- // standalone case, where the data node never emits a frame at all.
- SetStreamWireModeRaw(false)
- encBefore, decBefore := StreamFrameEncodedCount(), StreamFrameDecodedCount()
- body, err := c.Marshal(&streamv1.QueryResponse{})
- require.NoError(t, err)
- _, err = c.Unmarshal(body)
- require.NoError(t, err)
- require.Equal(t, encBefore, StreamFrameEncodedCount(), "proto encode must not count as a frame")
- require.Equal(t, decBefore, StreamFrameDecodedCount(), "proto decode must not count as a frame")
-
- // Flag ON with a frame body: the decode counts, and the send path's
- // IncrFrameEncoded counts the matching encode.
+ // A frame body: the decode counts, and the send path's IncrFrameEncoded counts
+ // the matching encode.
SetStreamWireModeRaw(true)
defer SetStreamWireModeRaw(false)
- encBefore, decBefore = StreamFrameEncodedCount(), StreamFrameDecodedCount()
- body, err = c.Marshal(frame)
+ encBefore, decBefore := StreamFrameEncodedCount(), StreamFrameDecodedCount()
+ body, err := c.Marshal(frame)
require.NoError(t, err)
_, err = c.Unmarshal(body)
require.NoError(t, err)
@@ -68,8 +57,8 @@ func TestStreamFrameCounters_CountRawTrafficOnly(t *testing.T) {
IncrFrameEncoded(TopicStreamQuery)
require.Equal(t, encBefore+1, StreamFrameEncodedCount())
- // Flag ON but a proto body still travels the proto path (the fallback a traced
- // query or an unsupported shape takes), so it must not be counted either.
+ // A proto body still travels the proto path (the fallback a traced query or an
+ // unsupported shape takes), so it must not be counted.
encBefore, decBefore = StreamFrameEncodedCount(), StreamFrameDecodedCount()
body, err = c.Marshal(&streamv1.QueryResponse{})
require.NoError(t, err)
@@ -86,17 +75,20 @@ func TestTraceFrameCounters_CountRawTrafficOnly(t *testing.T) {
}
frame := []byte{RawFrameMagicLeadingByte, 0x0a, 0x0b}
- SetTraceWireModeRaw(false)
+ SetTraceWireModeRaw(true)
+ defer SetTraceWireModeRaw(false)
+
+ // A proto response travels the proto fallback and must not be counted.
encBefore, decBefore := TraceFrameEncodedCount(), TraceFrameDecodedCount()
- body, err := c.Marshal(&tracev1.InternalQueryResponse{})
+ body, err := c.Marshal(&tracev1.InternalQueryResponse{
+ InternalTraces: []*tracev1.InternalTrace{{TraceId: "trace-1"}},
+ })
require.NoError(t, err)
_, err = c.Unmarshal(body)
require.NoError(t, err)
require.Equal(t, encBefore, TraceFrameEncodedCount(), "proto encode must not count as a frame")
require.Equal(t, decBefore, TraceFrameDecodedCount(), "proto decode must not count as a frame")
- SetTraceWireModeRaw(true)
- defer SetTraceWireModeRaw(false)
encBefore, decBefore = TraceFrameEncodedCount(), TraceFrameDecodedCount()
body, err = c.Marshal(frame)
require.NoError(t, err)
diff --git a/api/data/stream_frame_test.go b/api/data/stream_frame_test.go
index 4d7345366..dccbd178a 100644
--- a/api/data/stream_frame_test.go
+++ b/api/data/stream_frame_test.go
@@ -67,41 +67,28 @@ func TestStreamCodec_RawEmptyBody(t *testing.T) {
require.Empty(t, decoded.GetElements())
}
-// TestStreamCodec_FlagOffIsProto verifies flag-off encodes byte-identically to
-// plain proto.Marshal and round-trips through the proto path.
-func TestStreamCodec_FlagOffIsProto(t *testing.T) {
- SetStreamWireModeRaw(false)
- c := newStreamCodec()
- orig := &streamv1.QueryResponse{
- Elements: []*streamv1.Element{{ElementId: "element-1", Timestamp: nil}},
- }
-
- body, err := c.Marshal(orig)
- require.NoError(t, err)
- want, err := proto.Marshal(orig)
- require.NoError(t, err)
- require.Equal(t, want, body, "flag-off must encode byte-identically to proto.Marshal")
-
- got, err := c.Unmarshal(body)
- require.NoError(t, err)
- require.True(t, proto.Equal(orig, got.(*streamv1.QueryResponse)))
-}
-
-// TestStreamCodec_FlagOnProtoFallback verifies that flag-on, a proto.Message
-// value (e.g. the tracing path) still goes through the proto path.
-func TestStreamCodec_FlagOnProtoFallback(t *testing.T) {
+// TestStreamCodec_ProtoFallback verifies that a proto.Message value (the tracing
+// path, or any shape the columnar egress declines) still encodes byte-identically
+// to plain proto.Marshal and round-trips through the proto path. The non-empty
+// case matters on its own: it produces a body Unmarshal must route to proto
+// despite the raw-frame magic being 0x00, which an empty body never exercises.
+func TestStreamCodec_ProtoFallback(t *testing.T) {
SetStreamWireModeRaw(true)
defer SetStreamWireModeRaw(false)
c := newStreamCodec()
- orig := &streamv1.QueryResponse{}
-
- body, err := c.Marshal(orig)
- require.NoError(t, err)
- want, err := proto.Marshal(orig)
- require.NoError(t, err)
- require.Equal(t, want, body)
- got, err := c.Unmarshal(body)
- require.NoError(t, err)
- require.True(t, proto.Equal(orig, got.(*streamv1.QueryResponse)))
+ for _, orig := range []*streamv1.QueryResponse{
+ {},
+ {Elements: []*streamv1.Element{{ElementId: "element-1", Timestamp: nil}}},
+ } {
+ body, err := c.Marshal(orig)
+ require.NoError(t, err)
+ want, err := proto.Marshal(orig)
+ require.NoError(t, err)
+ require.Equal(t, want, body, "proto fallback must encode byte-identically to proto.Marshal")
+
+ got, err := c.Unmarshal(body)
+ require.NoError(t, err)
+ require.True(t, proto.Equal(orig, got.(*streamv1.QueryResponse)))
+ }
}
diff --git a/api/data/trace_frame_test.go b/api/data/trace_frame_test.go
index 7a8d125c0..8719eeb80 100644
--- a/api/data/trace_frame_test.go
+++ b/api/data/trace_frame_test.go
@@ -67,43 +67,30 @@ func TestTraceCodec_RawEmptyBody(t *testing.T) {
require.Empty(t, decoded.GetInternalTraces())
}
-// TestTraceCodec_FlagOffIsProto verifies flag-off encodes byte-identically to
-// plain proto.Marshal and round-trips through the proto path.
-func TestTraceCodec_FlagOffIsProto(t *testing.T) {
- SetTraceWireModeRaw(false)
- c := newTraceCodec()
- orig := &tracev1.InternalQueryResponse{
- InternalTraces: []*tracev1.InternalTrace{
- {TraceId: "trace-1", Key: -42, Spans: []*tracev1.Span{{SpanId: "span-1", Span: []byte("payload-1")}}},
- },
- }
-
- body, err := c.Marshal(orig)
- require.NoError(t, err)
- want, err := proto.Marshal(orig)
- require.NoError(t, err)
- require.Equal(t, want, body, "flag-off must encode byte-identically to proto.Marshal")
-
- got, err := c.Unmarshal(body)
- require.NoError(t, err)
- require.True(t, proto.Equal(orig, got.(*tracev1.InternalQueryResponse)))
-}
-
-// TestTraceCodec_FlagOnProtoFallback verifies that flag-on, a proto.Message
-// value (e.g. the tracing path) still goes through the proto path.
-func TestTraceCodec_FlagOnProtoFallback(t *testing.T) {
+// TestTraceCodec_ProtoFallback verifies that a proto.Message value (the tracing
+// path, or any shape the columnar egress declines) still encodes byte-identically
+// to plain proto.Marshal and round-trips through the proto path. The non-empty
+// case matters on its own: it produces a body Unmarshal must route to proto
+// despite the raw-frame magic being 0x00, which an empty body never exercises.
+func TestTraceCodec_ProtoFallback(t *testing.T) {
SetTraceWireModeRaw(true)
defer SetTraceWireModeRaw(false)
c := newTraceCodec()
- orig := &tracev1.InternalQueryResponse{}
-
- body, err := c.Marshal(orig)
- require.NoError(t, err)
- want, err := proto.Marshal(orig)
- require.NoError(t, err)
- require.Equal(t, want, body)
- got, err := c.Unmarshal(body)
- require.NoError(t, err)
- require.True(t, proto.Equal(orig, got.(*tracev1.InternalQueryResponse)))
+ for _, orig := range []*tracev1.InternalQueryResponse{
+ {},
+ {InternalTraces: []*tracev1.InternalTrace{
+ {TraceId: "trace-1", Key: -42, Spans: []*tracev1.Span{{SpanId: "span-1", Span: []byte("payload-1")}}},
+ }},
+ } {
+ body, err := c.Marshal(orig)
+ require.NoError(t, err)
+ want, err := proto.Marshal(orig)
+ require.NoError(t, err)
+ require.Equal(t, want, body, "proto fallback must encode byte-identically to proto.Marshal")
+
+ got, err := c.Unmarshal(body)
+ require.NoError(t, err)
+ require.True(t, proto.Equal(orig, got.(*tracev1.InternalQueryResponse)))
+ }
}
diff --git a/api/proto/banyandb/measure/v1/query.proto b/api/proto/banyandb/measure/v1/query.proto
index feadeb8ed..914aa94ea 100644
--- a/api/proto/banyandb/measure/v1/query.proto
+++ b/api/proto/banyandb/measure/v1/query.proto
@@ -84,7 +84,7 @@ message InternalQueryResponse {
repeated InternalDataPoint data_points = 1;
// trace contains the trace information of the query when trace is enabled
common.v1.Trace trace = 2;
- // raw_frame_body contains the vectorized raw frame body when raw wire mode is enabled
+ // raw_frame_body contains the vectorized raw frame body
bytes raw_frame_body = 3;
}
diff --git a/banyand/dquery/measure.go b/banyand/dquery/measure.go
index f6417c010..86670ff4d 100644
--- a/banyand/dquery/measure.go
+++ b/banyand/dquery/measure.go
@@ -24,7 +24,6 @@ import (
"time"
"github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
@@ -33,8 +32,6 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/query"
"github.com/apache/skywalking-banyandb/pkg/query/executor"
- "github.com/apache/skywalking-banyandb/pkg/query/logical"
- logical_measure "github.com/apache/skywalking-banyandb/pkg/query/logical/measure"
"github.com/apache/skywalking-banyandb/pkg/query/tracelabels"
vmeasure "github.com/apache/skywalking-banyandb/pkg/query/vectorized/measure"
vecplan "github.com/apache/skywalking-banyandb/pkg/query/vectorized/measure/plan"
@@ -69,20 +66,18 @@ func (p *measureQueryProcessor) Rev(ctx context.Context, message bus.Message) (r
e.RawJSON("req", logger.Proto(queryCriteria)).Msg("received a query event")
}
- schemas, measureSchemas, measureIndexRules, vecCfg, loadErr := p.loadMeasureSchemas(queryCriteria)
+ measureSchemas, measureIndexRules, vecCfg, loadErr := p.loadMeasureSchemas(queryCriteria)
if loadErr != nil {
resp = bus.NewMessage(bus.MessageID(now), common.NewError("%s", loadErr.Error()))
return
}
- // Operator-configured broadcast timeout overrides each plan's historical
+ // Operator-configured broadcast timeout overrides the plan's historical
// 15 s default. Setting it on vecCfg.BroadcastTimeout flows through
- // vecplan.AnalyzeDistributed into DistributedPlan.broadcastTimeout(); the
- // row path receives it as an explicit parameter on DistributedAnalyze.
- // Either way the two plans stay lockstep with the same liaison config.
+ // vecplan.AnalyzeDistributed into DistributedPlan.broadcastTimeout().
if p.broadcastTimeout > 0 {
vecCfg.BroadcastTimeout = p.broadcastTimeout
}
- plan, err := p.analyzeDistributedPlan(queryCriteria, schemas, measureSchemas, measureIndexRules, vecCfg)
+ plan, err := p.analyzeDistributedPlan(queryCriteria, measureSchemas, measureIndexRules, vecCfg)
if err != nil {
resp = bus.NewMessage(bus.MessageID(now), common.NewError("fail to analyze the query request for measure %s: %v", queryCriteria.Name, err))
return
@@ -176,9 +171,8 @@ func (p *measureQueryProcessor) Rev(ctx context.Context, message bus.Message) (r
}
func (p *measureQueryProcessor) loadMeasureSchemas(queryCriteria *measurev1.QueryRequest) (
- []logical.Schema, []*databasev1.Measure, [][]*databasev1.IndexRule, vmeasure.VectorizedConfig, error,
+ []*databasev1.Measure, [][]*databasev1.IndexRule, vmeasure.VectorizedConfig, error,
) {
- var schemas []logical.Schema
var measureSchemas []*databasev1.Measure
var measureIndexRules [][]*databasev1.IndexRule
var vecCfg vmeasure.VectorizedConfig
@@ -186,53 +180,31 @@ func (p *measureQueryProcessor) loadMeasureSchemas(queryCriteria *measurev1.Quer
meta := &commonv1.Metadata{Name: queryCriteria.Name, Group: g}
ec, measureErr := p.measureService.Measure(meta)
if measureErr != nil {
- return nil, nil, nil, vecCfg, fmt.Errorf("fail to get execution context for measure %s: %w", meta.GetName(), measureErr)
+ return nil, nil, vecCfg, fmt.Errorf("fail to get execution context for measure %s: %w", meta.GetName(), measureErr)
}
- // nolint:staticcheck // SA1019 - row schema is still used for the flag-off rollback path.
- s, schemaErr := logical_measure.BuildSchema(ec.GetSchema(), ec.GetIndexRules())
- if schemaErr != nil {
- return nil, nil, nil, vecCfg, fmt.Errorf("fail to build schema for measure %s: %w", meta.GetName(), schemaErr)
- }
- schemas = append(schemas, s)
measureSchemas = append(measureSchemas, ec.GetSchema())
measureIndexRules = append(measureIndexRules, ec.GetIndexRules())
- if vecEC, ok := ec.(measureVectorizedExecutionContext); ok {
- if groupIdx == 0 {
- vecCfg = vecEC.VectorizedConfig()
- }
- } else if data.MeasureWireModeRaw() {
- return nil, nil, nil, vecCfg, fmt.Errorf("measure %s is not vectorized-capable under raw wire mode", meta.GetName())
+ vecEC, ok := ec.(measureVectorizedExecutionContext)
+ if !ok {
+ return nil, nil, vecCfg, fmt.Errorf("measure %s is not vectorized-capable", meta.GetName())
+ }
+ if groupIdx == 0 {
+ vecCfg = vecEC.VectorizedConfig()
}
}
- return schemas, measureSchemas, measureIndexRules, vecCfg, nil
+ return measureSchemas, measureIndexRules, vecCfg, nil
}
-// analyzeDistributedPlan picks the vec plan when raw wire mode is on, otherwise
-// falls back to the row distributed plan (kill-switch / rollback rail).
-// See docs/operation/troubleshooting/measure-vec-flag-off-rollback.md.
func (p *measureQueryProcessor) analyzeDistributedPlan(
queryCriteria *measurev1.QueryRequest,
- schemas []logical.Schema,
measureSchemas []*databasev1.Measure,
measureIndexRules [][]*databasev1.IndexRule,
vecCfg vmeasure.VectorizedConfig,
) (measureDistributedExecutable, error) {
- if data.MeasureWireModeRaw() {
- if len(measureSchemas) == 0 {
- return nil, fmt.Errorf("vec distributed plan requires at least one measure schema")
- }
- return vecplan.AnalyzeDistributed(queryCriteria, measureSchemas, measureIndexRules, vecCfg)
- }
- // nolint:staticcheck // SA1019 - row distributed plan is the flag-off rollback path only.
- rowPlan, analyzeErr := logical_measure.DistributedAnalyze(queryCriteria, schemas, p.broadcastTimeout)
- if analyzeErr != nil {
- return nil, analyzeErr
- }
- plan, ok := rowPlan.(measureDistributedExecutable)
- if !ok {
- return nil, fmt.Errorf("distributed measure plan %T is not executable", rowPlan)
+ if len(measureSchemas) == 0 {
+ return nil, fmt.Errorf("vec distributed plan requires at least one measure schema")
}
- return plan, nil
+ return vecplan.AnalyzeDistributed(queryCriteria, measureSchemas, measureIndexRules, vecCfg)
}
func (p *measureQueryProcessor) buildNodeSelectors(queryCriteria *measurev1.QueryRequest, ml *logger.Logger) (map[string][]string, error) {
diff --git a/banyand/dquery/measure_test.go b/banyand/dquery/measure_test.go
index 6683c839e..b350ffac9 100644
--- a/banyand/dquery/measure_test.go
+++ b/banyand/dquery/measure_test.go
@@ -19,7 +19,6 @@ package dquery
import (
"testing"
- "github.com/apache/skywalking-banyandb/api/data"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
@@ -56,11 +55,10 @@ func rawWireModeTestSchema() *databasev1.Measure {
// measureQueryProcessor with a minimal fake measureService, invokes Rev
// with a crafted bus.Message, and asserts the concrete plan type on the
// returned MIterator. The plan-type gate here covers the AnalyzeDistributed
-// call site directly; the Rev routing boolean is covered by
-// TestRawWireMode_AlwaysUsesVecPlan.
+// call site directly.
func TestRawWireMode_PlanType_AnalyzeDistributed(t *testing.T) {
ms := rawWireModeTestSchema()
- cfg := vmeasure.VectorizedConfig{Enabled: true, BatchSize: 64, QueryMemoryMiB: 4}
+ cfg := vmeasure.VectorizedConfig{BatchSize: 64, QueryMemoryMiB: 4}
groupByTagProj := &modelv1.TagProjection{
TagFamilies: []*modelv1.TagProjection_TagFamily{
@@ -133,120 +131,3 @@ func TestRawWireMode_PlanType_AnalyzeDistributed(t *testing.T) {
})
}
}
-
-// TestRawWireMode_AlwaysUsesVecPlan asserts that under raw wire mode the
-// routing predicate (data.MeasureWireModeRaw()) is true for every request
-// shape, and false when the flag is off — regardless of request shape.
-// After Phase 6 the routing in Rev is a single boolean: data.MeasureWireModeRaw().
-func TestRawWireMode_AlwaysUsesVecPlan(t *testing.T) {
- shapes := []struct {
- req *measurev1.QueryRequest
- name string
- }{
- {
- name: "plain non-agg",
- req: &measurev1.QueryRequest{Groups: []string{"default"}},
- },
- {
- name: "orderby by index rule",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- OrderBy: &modelv1.QueryOrder{
- IndexRuleName: "idx_latency",
- Sort: modelv1.Sort_SORT_DESC,
- },
- },
- },
- {
- name: "multi-group",
- req: &measurev1.QueryRequest{Groups: []string{"a", "b"}},
- },
- {
- name: "top without agg",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- Top: &measurev1.QueryRequest_Top{
- Number: 5,
- FieldName: "value",
- FieldValueSort: modelv1.Sort_SORT_DESC,
- },
- },
- },
- {
- name: "raw groupby",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- GroupBy: &measurev1.QueryRequest_GroupBy{
- TagProjection: &modelv1.TagProjection{
- TagFamilies: []*modelv1.TagProjection_TagFamily{
- {Name: "default", Tags: []string{"service_id"}},
- },
- },
- },
- },
- },
- {
- name: "groupby + top",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- GroupBy: &measurev1.QueryRequest_GroupBy{
- TagProjection: &modelv1.TagProjection{
- TagFamilies: []*modelv1.TagProjection_TagFamily{
- {Name: "default", Tags: []string{"service_id"}},
- },
- },
- },
- Top: &measurev1.QueryRequest_Top{
- Number: 10,
- FieldName: "value",
- FieldValueSort: modelv1.Sort_SORT_DESC,
- },
- },
- },
- {
- name: "agg with groupby",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- Agg: &measurev1.QueryRequest_Aggregation{
- Function: modelv1.AggregationFunction_AGGREGATION_FUNCTION_SUM,
- FieldName: "value",
- },
- GroupBy: &measurev1.QueryRequest_GroupBy{
- TagProjection: &modelv1.TagProjection{
- TagFamilies: []*modelv1.TagProjection_TagFamily{
- {Name: "default", Tags: []string{"service_id"}},
- },
- },
- },
- },
- },
- {
- name: "agg only scalar reduce",
- req: &measurev1.QueryRequest{
- Groups: []string{"default"},
- Agg: &measurev1.QueryRequest_Aggregation{
- Function: modelv1.AggregationFunction_AGGREGATION_FUNCTION_MEAN,
- FieldName: "latency",
- },
- },
- },
- }
-
- prev := data.MeasureWireModeRaw()
- t.Cleanup(func() { data.SetMeasureWireModeRaw(prev) })
-
- for _, shape := range shapes {
- t.Run(shape.name+"/raw_on", func(t *testing.T) {
- data.SetMeasureWireModeRaw(true)
- if !data.MeasureWireModeRaw() {
- t.Fatal("routing predicate must be true when raw wire mode is on")
- }
- })
- t.Run(shape.name+"/raw_off", func(t *testing.T) {
- data.SetMeasureWireModeRaw(false)
- if data.MeasureWireModeRaw() {
- t.Fatal("routing predicate must be false when raw wire mode is off")
- }
- })
- }
-}
diff --git a/banyand/measure/block_batch_benchmark_test.go b/banyand/measure/block_batch_benchmark_test.go
index d4ae37a78..7f30abae0 100644
--- a/banyand/measure/block_batch_benchmark_test.go
+++ b/banyand/measure/block_batch_benchmark_test.go
@@ -143,7 +143,7 @@ type benchStorageHarness struct {
queryOpts queryOptions
}
-func setupBenchStorageHarness(b *testing.B) *benchStorageHarness {
+func setupBenchStorageHarness(b testing.TB) *benchStorageHarness {
b.Helper()
tmpPath, defFn := test.Space(require.New(b))
tst, err := newTSTable(
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index ae18e38ef..8dc28579c 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -21,6 +21,9 @@
package measure
import (
+ "errors"
+ "fmt"
+ "strconv"
"sync/atomic"
"time"
@@ -174,12 +177,35 @@ func (m *measure) VectorizedConfig() vmeasure.VectorizedConfig {
return m.vectorized
}
+// removedRowQueryFlag accepts --<engine>-vectorized-enabled=true as a no-op and
+// rejects =false, so an operator whose rollback command line survived the
+// upgrade gets a message naming the issue instead of "unknown flag". The flag
+// is removed entirely in 0.13.0.
+type removedRowQueryFlag string
+
+func (f *removedRowQueryFlag) Set(value string) error {
+ parsed, parseErr := strconv.ParseBool(value)
+ if parseErr != nil {
+ return fmt.Errorf("invalid boolean value %q: %w", value, parseErr)
+ }
+ if !parsed {
+ return errors.New("row-based query was removed in 0.12.0, see apache/skywalking#13998")
+ }
+ *f = removedRowQueryFlag(value)
+ return nil
+}
+
+func (f *removedRowQueryFlag) String() string { return string(*f) }
+
+func (*removedRowQueryFlag) Type() string { return "bool" }
+
// bindVectorizedFlags wires VectorizedConfig fields to a run.FlagSet. The
// defaults match vmeasure.DefaultConfig.
func bindVectorizedFlags(flagS *run.FlagSet, cfg *vmeasure.VectorizedConfig) {
defaults := vmeasure.DefaultConfig()
- flagS.BoolVar(&cfg.Enabled, "measure-vectorized-enabled", defaults.Enabled,
- "enable the vectorized measure query path")
+ removed := removedRowQueryFlag("true")
+ flagS.Var(&removed, "measure-vectorized-enabled", "deprecated no-op; removed in 0.13.0")
+ flagS.Lookup("measure-vectorized-enabled").NoOptDefVal = "true"
flagS.IntVar(&cfg.BatchSize, "measure-vectorized-batch-size", defaults.BatchSize,
"row count per vectorized batch")
flagS.IntVar(&cfg.QueryMemoryMiB, "measure-vectorized-query-memory-mib", defaults.QueryMemoryMiB,
diff --git a/banyand/measure/query_batch_top_test.go b/banyand/measure/query_batch_top_test.go
new file mode 100644
index 000000000..487c17a93
--- /dev/null
+++ b/banyand/measure/query_batch_top_test.go
@@ -0,0 +1,130 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query/model"
+ "github.com/apache/skywalking-banyandb/pkg/query/vectorized"
+)
+
+// TestApplyTopNOptionsOnlyForTopNSchema pins the gate that keeps an ordinary
+// Top query on the batch path. PullBatch falls back to the row merge only when
+// topNQueryOptions is set, and applyTopNOptions sets it for the internal
+// _top_n_result measure alone. A client Top clause never reaches it: the
+// analyzer routes Top through the vectorized BatchTop operator instead.
+//
+// Widening this gate to fire on a client Top clause would silently put ordinary
+// multi-block Top queries back on the row merge, which is what this asserts against.
+func TestApplyTopNOptionsOnlyForTopNSchema(t *testing.T) {
+ tests := []struct {
+ name string
+ measureName string
+ wantSet bool
+ }{
+ {name: "ordinary measure stays batch-native", measureName: "sw_metric", wantSet: false},
+ {name: "empty name stays batch-native", measureName: "", wantSet: false},
+ {name: "TopN pre-aggregation measure opts in", measureName: TopNSchemaName, wantSet: true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var result queryResult
+ applyTopNOptions(model.MeasureQueryOptions{
+ Name: tt.measureName,
+ Sort: modelv1.Sort_SORT_DESC,
+ Number: 10,
+ TopNFieldType: 0,
+ }, &result)
+ if tt.wantSet {
+ require.NotNil(t, result.topNQueryOptions, "the TopN pre-aggregation measure must opt into the row merge")
+ return
+ }
+ require.Nil(t, result.topNQueryOptions,
+ "measure %q must not opt into the row merge; a client Top clause goes through BatchTop", tt.measureName)
+ })
+ }
+}
+
+// TestPullBatchMultiBlockStaysBatchNative drives PullBatch over a query result
+// spanning more than one block cursor and asserts it merges natively, returning
+// every row in timestamp order without the row-path detour.
+func TestPullBatchMultiBlockStaysBatchNative(t *testing.T) {
+ h := setupBenchStorageHarness(t)
+
+ var qr queryResult
+ qr.ctx = context.TODO()
+ // mergeBatch resolves tags from batchSchema.Columns and the block cursor; only the
+ // row fallback reads tagProjection. Leaving it nil means a regression that routes
+ // this query through qr.merge null-fills every tag cell, which the per-batch
+ // assertion below rejects. Without this the test passes on either path.
+ qr.tagProjection = nil
+ qr.batchSchema = benchNativeSchema()
+ qr.orderByTS = true
+ qr.ascTS = true
+
+ ti := &tstIter{}
+ ti.init(h.parts, h.sids, h.queryOpts.minTimestamp, h.queryOpts.maxTimestamp)
+ for ti.nextBlock() {
+ bc := generateBlockCursor()
+ p := ti.piHeap[0]
+ opts := h.queryOpts
+ opts.TagProjection = benchStorageProj
+ opts.FieldProjection = benchStorageFieldProj
+ bc.init(p.p, p.curBlock, opts)
+ qr.data = append(qr.data, bc)
+ }
+ defer qr.Release()
+
+ require.Nil(t, qr.topNQueryOptions, "an ordinary query must not carry TopN options")
+
+ var rows int
+ var lastTS int64
+ var checkedMultiBlock bool
+ for {
+ batch, err := qr.PullBatch(context.TODO())
+ require.NoError(t, err)
+ if batch == nil {
+ break
+ }
+ if !checkedMultiBlock {
+ require.Greater(t, len(qr.data), 1,
+ "the merge must run over more than one loaded cursor, not the single-block fast path")
+ checkedMultiBlock = true
+ }
+ require.NotEmpty(t, batch.Timestamps, "a non-nil batch must carry rows")
+ for _, ts := range batch.Timestamps {
+ require.GreaterOrEqual(t, ts, lastTS, "ascending timestamp order must hold across the block merge")
+ lastTS = ts
+ }
+ svc, ok := batch.Tags[0].(*vectorized.TypedColumn[string])
+ require.True(t, ok, "the native schema must decode svc into a string column")
+ for idx, value := range svc.Data() {
+ require.False(t, svc.IsNull(idx), "the batch merge must fill svc from the block cursor")
+ require.Equal(t, "alpha", value)
+ }
+ rows += batch.RowCount()
+ batch.Release()
+ }
+ require.Equal(t, benchStorageSeries*benchStorageRowsPer, rows,
+ "the native merge must return every row across all blocks")
+}
diff --git a/banyand/measure/svc_data.go b/banyand/measure/svc_data.go
index 8cd95ed5c..dfebb524e 100644
--- a/banyand/measure/svc_data.go
+++ b/banyand/measure/svc_data.go
@@ -242,15 +242,12 @@ func (s *dataSVC) PreRun(ctx context.Context) error {
s.l.Info().Msg("memory protector is initialized in PreRun")
// Publish the per-process wire mode for TopicInternalMeasureQuery so
// the queue's per-topic ResponseCodec dispatcher selects RawFrameCodec
- // when this process is flag-on (G9f.5.b's vec raw columnar frame body
- // path emits the body as []byte; sub.go passes it through under
- // flag-on) and ProtoCodec when flag-off. The data-node Rev now
- // branches on data.MeasureWireModeRaw() + the vec MIterator's
- // RawFrameSource capability, so this flip lands the synchronized
- // cutover that G9f.5.a's runbook describes.
- data.SetMeasureWireModeRaw(s.option.vectorized.Enabled)
+ // for the vec raw columnar frame body (G9f.5.b emits the body as
+ // []byte; sub.go passes it through). The data-node Rev branches on
+ // data.MeasureWireModeRaw() + the vec MIterator's RawFrameSource
+ // capability.
+ data.SetMeasureWireModeRaw(true)
s.l.Info().
- Bool("measure_vectorized_enabled", s.option.vectorized.Enabled).
Bool("measure_wire_mode_raw", data.MeasureWireModeRaw()).
Msg("G9f wire mode published for TopicInternalMeasureQuery (data svc)")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
diff --git a/banyand/measure/svc_liaison.go b/banyand/measure/svc_liaison.go
index 7737ee0c7..ae77cebd2 100644
--- a/banyand/measure/svc_liaison.go
+++ b/banyand/measure/svc_liaison.go
@@ -153,14 +153,10 @@ func (s *liaison) PreRun(ctx context.Context) error {
// Publish the per-process wire mode for TopicInternalMeasureQuery.
// On the liaison this controls the DECODE side: an incoming response
// body on TopicInternalMeasureQuery is parsed as raw frame bytes
- // under flag-on (G9f.5.c's distributedPlan recognizes the []byte
- // shape and runs ReduceFramesToInternalDataPoints /
- // DecodeFramesToInternalDataPoints) or proto under flag-off. The
- // per-process modes MUST match across the cluster — the runbook
- // covers detection + recovery for a partial-rollout skew.
- data.SetMeasureWireModeRaw(s.option.vectorized.Enabled)
+ // (G9f.5.c's distributed plan recognizes the []byte shape and runs
+ // ReduceFramesToInternalDataPoints / DecodeFramesToInternalDataPoints).
+ data.SetMeasureWireModeRaw(true)
s.l.Info().
- Bool("measure_vectorized_enabled", s.option.vectorized.Enabled).
Bool("measure_wire_mode_raw", data.MeasureWireModeRaw()).
Msg("G9f wire mode published for TopicInternalMeasureQuery (liaison)")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
diff --git a/banyand/measure/svc_standalone.go b/banyand/measure/svc_standalone.go
index a35d42caa..4ccc5f7cd 100644
--- a/banyand/measure/svc_standalone.go
+++ b/banyand/measure/svc_standalone.go
@@ -251,18 +251,13 @@ func (s *standalone) PreRun(ctx context.Context) error {
s.l = logger.GetLogger(s.Name())
s.l.Info().Msg("memory protector is initialized in PreRun")
// Publish the per-process wire mode for TopicInternalMeasureQuery so the
- // queue's per-topic ResponseCodec dispatcher selects RawFrameCodec when
- // this process is flag-on (vec raw columnar frame body) and ProtoCodec
- // when flag-off — topic-AND-process-wire-mode selection, G9f spec G9f.0.
- // Flags have been parsed by the time PreRun fires.
- data.SetMeasureWireModeRaw(s.option.vectorized.Enabled)
- // Loud announcement so operators see the cluster wire mode at boot.
- // G9f is a hard cutover — a botched partial rollout (some nodes
- // flag-on, some flag-off) is the load-bearing failure mode the runbook
- // covers. Surfacing the mode at startup makes the rollout state
- // auditable from logs (G9f spec Principle 3 — fail loud).
+ // queue's per-topic ResponseCodec dispatcher selects RawFrameCodec for
+ // the vec raw columnar frame body — topic-AND-process-wire-mode
+ // selection, G9f spec G9f.0.
+ data.SetMeasureWireModeRaw(true)
+ // Loud announcement so operators see the cluster wire mode at boot,
+ // making the rollout state auditable from logs.
s.l.Info().
- Bool("measure_vectorized_enabled", s.option.vectorized.Enabled).
Bool("measure_wire_mode_raw", data.MeasureWireModeRaw()).
Msg("G9f wire mode published for TopicInternalMeasureQuery")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
diff --git a/banyand/query/processor.go b/banyand/query/processor.go
index 32e483117..bdea9490e 100644
--- a/banyand/query/processor.go
+++ b/banyand/query/processor.go
@@ -170,52 +170,41 @@ func (p *streamQueryProcessor) Rev(ctx context.Context, message bus.Message) (re
}()
}
- // Vec dispatch: attempt the native columnar path before the row execution.
- // Gated on the engine flag (VectorizedConfig().Enabled). The plan must expose
- // StreamVecExecutable (single localIndexScan, no tag filter / multi-group /
- // skipping filter) — otherwise the assertion fails and we fall through to the
- // row path below. When tracing is on we MUST return a proto QueryResponse (it
- // carries common.v1.Trace); the frame emit is gated on tracer == nil.
- if p.streamService.VectorizedConfig().Enabled {
- if handled, vecResp := p.tryStreamVecDispatch(ctx, plan, queryCriteria, tracer != nil); handled {
- resp = vecResp
- se := plan.(executor.StreamExecutable)
- se.Close()
- return
- }
- }
-
- se := plan.(executor.StreamExecutable)
- defer se.Close()
- entities, err := se.Execute(ctx)
- if err != nil {
- p.log.Error().Err(err).RawJSON("req", logger.Proto(queryCriteria)).Msg("fail to execute the query plan")
- resp = bus.NewMessage(bus.MessageID(now), common.NewError("execute the query plan for stream %s: %v", queryCriteria.GetName(), err))
+ // Vec dispatch is the only stream execution path; the row path was removed in
+ // 0.12.0. Every analyzed plan shape must therefore be vec-eligible, so a decline
+ // is a hard error rather than a silent proto fallback (same no-silent-fallback
+ // discipline as measure and trace). When tracing is on we MUST return a proto
+ // QueryResponse (it carries common.v1.Trace); the frame emit is gated on tracer == nil.
+ handled, vecResp := p.tryStreamVecDispatch(ctx, plan, queryCriteria, tracer != nil)
+ // Read the decline reason off the plan before Close, so this does not depend on
+ // Close staying a no-op for every stream node.
+ var declineReason string
+ if !handled {
+ declineReason = logical_stream.VecDeclineReason(plan)
+ }
+ plan.(executor.StreamCloser).Close()
+ if !handled {
+ p.log.Error().Str("plan", plan.String()).Str("reason", declineReason).
+ RawJSON("req", logger.Proto(queryCriteria)).Msg("no vectorized execution path for the query plan")
+ resp = bus.NewMessage(bus.MessageID(now), common.NewError(
+ "no execution path for stream %s: %s; row execution was removed in 0.12.0 (apache/skywalking#13998)",
+ queryCriteria.GetName(), declineReason))
return
}
-
- resp = bus.NewMessage(bus.MessageID(now), &streamv1.QueryResponse{Elements: entities})
-
- if !queryCriteria.Trace && p.slowQuery > 0 {
- latency := time.Since(n)
- if latency > p.slowQuery {
- p.log.Warn().Dur("latency", latency).RawJSON("req", logger.Proto(queryCriteria)).Int("resp_count", len(entities)).Msg("stream slow query")
- }
- }
+ resp = vecResp
return
}
-// tryStreamVecDispatch attempts the native columnar (vec) path for a stream
-// query. It returns (false, nil) when the plan is not vec-eligible (the caller
-// then runs the row path). On success it returns (true, resp) where resp is:
+// tryStreamVecDispatch runs the native columnar (vec) path for a stream query.
+// It returns (false, nil) when the plan is not vec-eligible, which the caller
+// treats as a hard error. On success it returns (true, resp) where resp is:
// - a []byte columnar frame body, when the process is a distributed data node
// with the raw wire mode on and tracing off (traced == false);
// - a *streamv1.QueryResponse with materialized Elements, for standalone, for a
// traced query (the frame has no trace channel), or when the wire mode is off.
//
-// Any error is surfaced as a *common.Error response (handled=true) rather than
-// falling back to row, matching the measure/trace no-silent-fallback discipline
-// once we have committed to vec.
+// Any error is surfaced as a *common.Error response (handled=true), matching the
+// measure/trace no-silent-fallback discipline.
func (p *streamQueryProcessor) tryStreamVecDispatch(ctx context.Context, plan logical.Plan,
queryCriteria *streamv1.QueryRequest, traced bool,
) (bool, bus.Message) {
@@ -252,7 +241,7 @@ func (p *streamQueryProcessor) tryStreamVecDispatch(ctx context.Context, plan lo
// so the send path passes it through and the liaison decodes it. The liaison
// applies the global offset/limit slice, so the data node emits the whole
// (already per-node-capped) batch set — no slice here.
- if !hasFilter && streamVecEmitAsFrame(p.distributed, data.StreamWireModeRaw(), traced) {
+ if !hasFilter && !vecExec.HidesOrderTag() && streamVecEmitAsFrame(p.distributed, data.StreamWireModeRaw(), traced) {
merged, mergeErr := mergeStreamBatches(schema, batches)
if mergeErr != nil {
p.log.Error().Err(mergeErr).RawJSON("req", logger.Proto(queryCriteria)).Msg("fail to merge the vectorized stream batches")
@@ -428,7 +417,6 @@ func buildMeasureContext(measureService measure.Service, log *logger.Logger, que
if ecErr != nil {
return nil, fmt.Errorf("fail to get execution context for measure %s: %w", meta.GetName(), ecErr)
}
- // nolint:staticcheck // SA1019 — row-path BuildSchema is the only production path until G8 ships.
s, schemaErr := logical_measure.BuildSchema(ec.GetSchema(), ec.GetIndexRules())
if schemaErr != nil {
return nil, fmt.Errorf("fail to build schema for measure %s: %w", meta.GetName(), schemaErr)
@@ -459,12 +447,6 @@ type vecExecutionContext interface {
// executeMeasurePlan executes the measure query plan and returns the iterator.
//
-// G8d: the vec subsystem is tried first via vecplan.Dispatch. When the
-// request is vec-eligible (no GroupBy/Agg/Top, has TimeRange, no hidden
-// criteria tags), dispatch returns a vec MIterator and the row-path
-// Analyze is skipped entirely. Otherwise, control flows through to the
-// deprecated row plan unchanged.
-//
// The second return value is the rendered plan string used by the caller
// for tracing — abstracted to a string so the vec subsystem (which does
// not produce a logical.Plan) can participate.
@@ -474,107 +456,65 @@ func executeMeasurePlan(
mctx *measureExecutionContext,
emitPartial bool,
) (executor.MIterator, string, error) {
- if mit, planStr, handled, dispatchErr := tryVecDispatch(ctx, queryCriteria, mctx, emitPartial); dispatchErr != nil {
+ mit, planStr, dispatchErr := dispatchMeasure(ctx, queryCriteria, mctx, emitPartial)
+ if dispatchErr != nil {
return nil, "", fmt.Errorf("fail to dispatch the query request for measure %s: %w", queryCriteria.GetName(), dispatchErr)
- } else if handled {
- if e := mctx.ml.Debug(); e.Enabled() {
- e.Str("plan", planStr).Msg("vec query plan")
- }
- return mit, planStr, nil
- }
-
- // nolint:staticcheck // SA1019 — row-path Analyze is the only production path until G8 ships.
- plan, planErr := logical_measure.Analyze(queryCriteria, mctx.metadata, mctx.schemas, mctx.ecc, emitPartial)
- if planErr != nil {
- return nil, "", fmt.Errorf("fail to analyze the query request for measure %s: %w", queryCriteria.GetName(), planErr)
}
if e := mctx.ml.Debug(); e.Enabled() {
- e.Str("plan", plan.String()).Msg("query plan")
- }
- mIterator, execErr := plan.(executor.MeasureExecutable).Execute(ctx)
- if execErr != nil {
- mctx.ml.Error().Err(execErr).RawJSON("req", logger.Proto(queryCriteria)).Msg("fail to query")
- return nil, "", fmt.Errorf("fail to execute the query plan for measure %s: %w", queryCriteria.GetName(), execErr)
+ e.Str("plan", planStr).Msg("vec query plan")
}
- return mIterator, plan.String(), nil
+ return mit, planStr, nil
}
-// tryVecDispatch is the thin adapter from measureExecutionContext into
+// dispatchMeasure is the thin adapter from measureExecutionContext into
// the vec dispatch inputs. Multi-measure queries (len(ecc) > 1) are
-// handled per-group: each group is dispatched to vec individually and
-// the per-group iterators are merged through the row path's exact
-// cross-group ordering via logical_measure.MergeGroupMIterators (G9f.1,
-// reusing the row sortableDataPoints + sortedMIterator stack so the
-// cross-group order and version dedup are reproduced by construction,
-// not by parallel reimplementation). If any group's executor context is
-// not vec-capable, or any group's dispatch declines to handle the
-// request, the whole multi-measure request falls through to the row
-// path — mixing vec for some groups with row for others would produce a
-// merged result the row path cannot validate.
+// handled per-group: each group is dispatched individually and the
+// per-group iterators are merged through the shared cross-group ordering
+// stack in logical_measure.MergeGroupMIterators (G9f.1, reusing
+// sortableDataPoints + sortedMIterator so the cross-group order and
+// version dedup are reproduced by construction, not by parallel
+// reimplementation).
//
// Distributed Map-mode GroupBy+Agg (emitPartial=true) routes to vec via
// AggModeMap (G9f.2): vecplan.Dispatch receives emitPartial and the
// BatchAggregation operator emits typed-column partials. As of G9f.4 the
// data-node side handles distributed Top-over-Agg too — the vec plan emits
// Scan → GroupByAgg(Map) → Top → Limit per node, BatchTop sorts on the
-// partial value column, and the row-path liaison's distributedPlan dedupes
-// + applies the global Top across the per-node partial top-Ns (mirrors the
-// row path's two-pass distributed Top approach; pushDownAgg is set whenever
-// Agg != nil, see measure_analyzer.go:179, and Top is applied AFTER
-// aggregation at :205-207).
-func tryVecDispatch(
+// partial value column, and the liaison's distributed plan dedupes and
+// applies the global Top across the per-node partial top-Ns.
+func dispatchMeasure(
ctx context.Context,
queryCriteria *measurev1.QueryRequest,
mctx *measureExecutionContext,
emitPartial bool,
-) (executor.MIterator, string, bool, error) {
- // Empty execution context = no measures to query. Fall through to row
- // in both flag states — the row path handles the degenerate empty
- // case identically (returns an empty MIterator). This is NOT a vec
- // fall-through under flag-on (we never entered vec); it is the
- // "no work to do" exit.
- if len(mctx.ecc) == 0 {
- return nil, "", false, nil
- }
+) (executor.MIterator, string, error) {
vecs := make([]vecExecutionContext, len(mctx.ecc))
- flagOn := false
for groupIdx, ec := range mctx.ecc {
vec, ok := ec.(vecExecutionContext)
if !ok {
- // A non-vec execution context can legitimately appear only on
- // the flag-off rollback path (production storage satisfies
- // vecExecutionContext). If ANY group's executor lacks the
- // capability and the cluster is flag-on, that is a botched
- // rollout and must fail loud — no proto/row fall-through.
- if flagOn {
- return nil, "", true, fmt.Errorf("vec dispatch: group %d execution context not vec-capable under flag-on (rollout skew?)", groupIdx)
- }
- return nil, "", false, nil
+ // Production storage always satisfies vecExecutionContext. A
+ // group that does not is a botched rollout and must fail loud.
+ return nil, "", fmt.Errorf("vec dispatch: group %d execution context not vec-capable (rollout skew?)", groupIdx)
}
vecs[groupIdx] = vec
- // Detect flag-on as soon as the first vec config is observable.
- // Per-process flag means every group's config agrees in practice.
- if groupIdx == 0 && vec.VectorizedConfig().Enabled {
- flagOn = true
- }
}
if len(mctx.ecc) == 1 {
return vecplan.Dispatch(ctx, queryCriteria, mctx.metadata[0], vecs[0].GetSchema(),
mctx.schemas[0], vecs[0], vecs[0].VectorizedConfig(), emitPartial, false)
}
// Multi-measure projection validation: a tag/field is valid if it
- // resolves in ANY group's schema (mirrors measure_analyzer.Analyze's
- // mergeSchema(ss) union). Running the per-group validation inside
- // Dispatch would reject the schema-evolution case where one group
- // added a tag/field the others lack (multi_group_new_tag_field
- // integration fixture). We pre-validate here against the union once,
- // then skip the per-group validation inside Dispatch.
+ // resolves in ANY group's schema (mirrors the mergeSchema(ss) union).
+ // Running the per-group validation inside Dispatch would reject the
+ // schema-evolution case where one group added a tag/field the others
+ // lack (multi_group_new_tag_field integration fixture). We pre-validate
+ // here against the union once, then skip the per-group validation
+ // inside Dispatch.
measureSchemas := make([]*databasev1.Measure, len(vecs))
for i, v := range vecs {
measureSchemas[i] = v.GetSchema()
}
if projErr := vecplan.ValidateMultiGroupProjection(queryCriteria, mctx.schemas, measureSchemas); projErr != nil {
- return nil, "", true, projErr
+ return nil, "", projErr
}
iters := make([]executor.MIterator, 0, len(mctx.ecc))
planStrs := make([]string, 0, len(mctx.ecc))
@@ -584,20 +524,11 @@ func tryVecDispatch(
}
}
for groupIdx, vec := range vecs {
- mit, planStr, handled, dispatchErr := vecplan.Dispatch(ctx, queryCriteria,
+ mit, planStr, dispatchErr := vecplan.Dispatch(ctx, queryCriteria,
mctx.metadata[groupIdx], vec.GetSchema(), mctx.schemas[groupIdx], vec, vec.VectorizedConfig(), emitPartial, true)
if dispatchErr != nil {
closeOpened()
- // handled=true so the caller surfaces the error rather than
- // retrying on row (no fall-through under flag-on).
- return nil, "", true, dispatchErr
- }
- if !handled {
- closeOpened()
- // Per-group vec config is flag-off (rollback rail). Forward
- // the rollback decision to the caller so the entire
- // multi-measure request runs row-side end-to-end.
- return nil, "", false, nil
+ return nil, "", dispatchErr
}
iters = append(iters, mit)
planStrs = append(planStrs, planStr)
@@ -605,10 +536,10 @@ func tryVecDispatch(
order, orderErr := logical_measure.ResolveCrossGroupMergeOrder(queryCriteria, mctx.schemas)
if orderErr != nil {
closeOpened()
- return nil, "", false, orderErr
+ return nil, "", orderErr
}
merged := logical_measure.MergeGroupMIterators(iters, order)
- return merged, fmt.Sprintf("vec-multi-measure%v", planStrs), true, nil
+ return merged, fmt.Sprintf("vec-multi-measure%v", planStrs), nil
}
// collectInternalDataPoints collects InternalDataPoints from the iterator.
@@ -1305,11 +1236,11 @@ func (p *traceQueryProcessor) executeQuery(ctx context.Context, queryCriteria *t
return
}
- // Native wire mode (flag-on, no tracing): emit a columnar frame body so the
- // send path passes it through as opaque bytes and the liaison decodes it
- // without the protobuf message-slice/oneof machinery. The tracing path keeps
- // the proto body (the traceMonitor defer needs *InternalQueryResponse).
- if p.distributed && data.TraceWireModeRaw() && traceMonitor == nil {
+ // Native wire mode (no tracing): emit a columnar frame body so the send path
+ // passes it through as opaque bytes and the liaison decodes it without the
+ // protobuf message-slice/oneof machinery. The tracing path keeps the proto
+ // body (the traceMonitor defer needs *InternalQueryResponse).
+ if p.distributed && traceMonitor == nil {
results, buildErr := p.buildFrameTraceResults(resultIterator, queryCriteria, execPlan)
if buildErr != nil {
p.log.Error().Err(buildErr).RawJSON("req", logger.Proto(queryCriteria)).Msg("fail to process trace results")
diff --git a/banyand/query/tracing_test.go b/banyand/query/tracing_test.go
index 6e5fea6b9..e70f191b7 100644
--- a/banyand/query/tracing_test.go
+++ b/banyand/query/tracing_test.go
@@ -68,30 +68,6 @@ func assertRootSpanFinalized(t *testing.T, traceProto *commonv1.Trace, label str
}
}
-// TestRootSpanDurationNonZero_DataNode_ProtoWire mirrors the proto-wire data-node
-// response path in measureInternalQueryProcessor.Rev (processor.go ~line 623-641):
-//
-// span.Tag(...)
-// span.Stop()
-// d.Trace = tracer.ToProto()
-//
-// The regression being guarded: if ToProto() is called before Stop(), the
-// captured span has EndTime=nil and Duration=0.
-func TestRootSpanDurationNonZero_DataNode_ProtoWire(t *testing.T) {
- ctx := context.Background()
- tracer, ctx := query.NewTracer(ctx, "proto-wire-test")
- span, _ := tracer.StartSpan(ctx, "data-node-proto")
-
- span.Tag("plan", "test-plan")
- span.Tag("resp_count", "42")
-
- // Proto-wire path: Stop then ToProto.
- span.Stop()
- traceProto := tracer.ToProto()
-
- assertRootSpanFinalized(t, traceProto, "proto-wire", false)
-}
-
// TestRootSpanDurationNonZero_DataNode_RawWire mirrors the raw-wire data-node
// success response path in measureInternalQueryProcessor.Rev (processor.go
// ~line 606-613), which is reached only when data.MeasureWireModeRaw() is true:
@@ -122,31 +98,16 @@ func TestRootSpanDurationNonZero_DataNode_RawWire(t *testing.T) {
assertRootSpanFinalized(t, traceProto, "raw-wire", false)
}
-// TestRootSpanDurationNonZero_DataNode_ErrorPath mirrors both error branches in
-// measureInternalQueryProcessor.Rev:
-//
-// 1. Proto-wire error (processor.go ~line 633-637):
-// span.Error(err); span.Stop(); resp = &InternalQueryResponse{Trace: tracer.ToProto()}
+// TestRootSpanDurationNonZero_DataNode_ErrorPath mirrors the raw-wire error
+// branches in measureInternalQueryProcessor.Rev (processor.go ~line 583-588 and
+// ~line 596-601):
//
-// 2. Raw-wire error (processor.go ~line 583-588 and ~line 596-601):
-// span.Error(err); span.Stop(); resp = &InternalQueryResponse{Trace: tracer.ToProto()}
+// span.Error(err); span.Stop(); resp = &InternalQueryResponse{Trace: tracer.ToProto()}
//
// Even on the error path the root span must have populated Duration and EndTime.
func TestRootSpanDurationNonZero_DataNode_ErrorPath(t *testing.T) {
simulatedErr := errors.New("simulated query execution error")
- t.Run("proto-wire-error", func(t *testing.T) {
- ctx := context.Background()
- tracer, ctx := query.NewTracer(ctx, "proto-wire-error-test")
- span, _ := tracer.StartSpan(ctx, "data-node-proto-error")
-
- span.Error(simulatedErr)
- span.Stop()
- traceProto := tracer.ToProto()
-
- assertRootSpanFinalized(t, traceProto, "proto-wire-error", true)
- })
-
t.Run("raw-wire-error", func(t *testing.T) {
prevMode := data.MeasureWireModeRaw()
data.SetMeasureWireModeRaw(true)
diff --git a/banyand/stream/benchmark_test.go b/banyand/stream/benchmark_test.go
index d7ae914d0..2c36679ee 100644
--- a/banyand/stream/benchmark_test.go
+++ b/banyand/stream/benchmark_test.go
@@ -20,20 +20,14 @@ package stream
import (
"context"
"crypto/rand"
- "io"
- "math"
"math/big"
"strconv"
- "sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/banyand/protector"
"github.com/apache/skywalking-banyandb/pkg/convert"
@@ -41,10 +35,6 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index/posting"
"github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- logicalstream "github.com/apache/skywalking-banyandb/pkg/query/logical/stream"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
)
const (
@@ -65,12 +55,6 @@ type parameter struct {
endTimestamp int
}
-var pList = [3]parameter{
- {batchCount: 2, timestampCount: 500, seriesCount: 100, tagCardinality: 10, startTimestamp: 1, endTimestamp: 1000, scenario: "large-scale"},
- {batchCount: 2, timestampCount: 500, seriesCount: 100, tagCardinality: 10, startTimestamp: 900, endTimestamp: 1000, scenario: "latest"},
- {batchCount: 2, timestampCount: 500, seriesCount: 100, tagCardinality: 10, startTimestamp: 300, endTimestamp: 400, scenario: "historical"},
-}
-
type mockIndex map[string]map[common.SeriesID]posting.List
func (mi mockIndex) insert(value string, seriesID common.SeriesID, timestamp int) {
@@ -83,34 +67,6 @@ func (mi mockIndex) insert(value string, seriesID common.SeriesID, timestamp int
mi[value][seriesID].Insert(uint64(timestamp))
}
-type mockFilter struct {
- index mockIndex
- value string
-}
-
-func (mf mockFilter) String() string {
- return "filter"
-}
-
-func (mf mockFilter) Execute(_ index.GetSearcher, seriesID common.SeriesID, _ *index.RangeOpts) (posting.List, posting.List, error) {
- return mf.index[mf.value][seriesID], roaring.DummyPostingList, nil
-}
-
-func (mf mockFilter) ShouldSkip(_ index.FilterOp) (bool, error) {
- return false, nil
-}
-
-type databaseSupplier struct {
- database atomic.Value
-}
-
-func (dbs *databaseSupplier) SupplyTSDB() io.Closer {
- if v := dbs.database.Load(); v != nil {
- return v.(io.Closer)
- }
- return nil
-}
-
func generateRandomNumber(maxValue int64) int {
n, _ := rand.Int(rand.Reader, big.NewInt(maxValue))
return int(n.Int64()) + 1
@@ -200,162 +156,3 @@ func openDatabase(b testing.TB, path string) storage.TSDB[*tsTable, option] {
require.NoError(b, err)
return db
}
-
-func write(b testing.TB, p parameter, esList []*elements, docsList []index.Documents) storage.TSDB[*tsTable, option] {
- // Initialize a tstIter object.
- tmpPath, defFn := test.Space(require.New(b))
- defer defFn()
-
- db := openDatabase(b, tmpPath)
- var docs index.Documents
- for i := 1; i <= p.seriesCount; i++ {
- entity := []*modelv1.TagValue{
- {
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: entityTagValuePrefix + strconv.Itoa(i),
- },
- },
- },
- }
- series := &pbv1.Series{
- Subject: "benchmark",
- EntityValues: entity,
- }
- err := series.Marshal()
- require.NoError(b, err)
- docs = append(docs, index.Document{
- DocID: uint64(i),
- EntityValues: series.Buffer,
- })
- }
- seg, err := db.CreateSegmentIfNotExist(time.Unix(0, esList[0].timestamps[0]))
- require.NoError(b, err)
- seg.IndexDB().Insert(docs)
-
- tst, err := seg.CreateTSTableIfNotExist(common.ShardID(0))
- require.NoError(b, err)
- for i := range esList {
- tst.mustAddElements(esList[i])
- tst.Index().Write(docsList[i])
- }
- return db
-}
-
-func generateStream(db storage.TSDB[*tsTable, option]) *stream {
- dbSupplier := &databaseSupplier{
- database: atomic.Value{},
- }
- dbSupplier.database.Store(db)
- entity := &databasev1.Entity{
- TagNames: []string{"entity-tag"},
- }
- tagFamily := &databasev1.TagFamilySpec{
- Name: "benchmark-family",
- Tags: []*databasev1.TagSpec{
- {
- Name: "entity-tag",
- Type: databasev1.TagType_TAG_TYPE_STRING,
- },
- {
- Name: "filter-tag",
- Type: databasev1.TagType_TAG_TYPE_STRING,
- },
- },
- }
- schema := &databasev1.Stream{
- Entity: entity,
- TagFamilies: []*databasev1.TagFamilySpec{tagFamily},
- }
- s := &stream{
- schema: schema,
- }
- s.tsdb.Store(db)
- return s
-}
-
-func generateStreamQueryOptions(p parameter, midx mockIndex) model.StreamQueryOptions {
- timeRange := timestamp.TimeRange{
- Start: time.Unix(int64(p.startTimestamp), 0),
- End: time.Unix(int64(p.endTimestamp), 0),
- IncludeStart: true,
- IncludeEnd: true,
- }
- entities := make([][]*modelv1.TagValue, 0)
- for i := 1; i <= p.seriesCount; i++ {
- entity := []*modelv1.TagValue{
- {
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: entityTagValuePrefix + strconv.Itoa(i),
- },
- },
- },
- }
- entities = append(entities, entity)
- }
- num := generateRandomNumber(int64(p.tagCardinality))
- value := filterTagValuePrefix + strconv.Itoa(num)
- filter := mockFilter{
- index: midx,
- value: value,
- }
- indexRule := &databasev1.IndexRule{
- Metadata: &commonv1.Metadata{
- Id: uint32(1),
- },
- Tags: []string{"filter-tag"},
- Type: databasev1.IndexRule_TYPE_INVERTED,
- }
- order := &index.OrderBy{
- Index: indexRule,
- Sort: modelv1.Sort_SORT_ASC,
- }
- tagProjection := model.TagProjection{
- Family: "benchmark-family",
- Names: []string{"entity-tag", "filter-tag"},
- }
- return model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &timeRange,
- Entities: entities,
- InvertedFilter: filter,
- SkippingFilter: nil,
- Order: order,
- TagProjection: []model.TagProjection{tagProjection},
- MaxElementSize: math.MaxInt32,
- }
-}
-
-func BenchmarkFilter(b *testing.B) {
- b.ReportAllocs()
- ctx := context.TODO()
- for _, p := range pList {
- esList, docsList, idx := generateData(p)
- db := write(b, p, esList, docsList)
- s := generateStream(db)
- sqo := generateStreamQueryOptions(p, idx)
- sqo.Order = nil
- b.Run("filter-"+p.scenario, func(b *testing.B) {
- res, err := s.Query(ctx, sqo)
- require.NoError(b, err)
- logicalstream.BuildElementsFromStreamResult(ctx, res, sqo.TagProjection)
- })
- }
-}
-
-func BenchmarkSort(b *testing.B) {
- b.ReportAllocs()
- ctx := context.TODO()
- for _, p := range pList {
- esList, docsList, idx := generateData(p)
- db := write(b, p, esList, docsList)
- s := generateStream(db)
- sqo := generateStreamQueryOptions(p, idx)
- b.Run("sort-"+p.scenario, func(b *testing.B) {
- res, err := s.Query(ctx, sqo)
- require.NoError(b, err)
- logicalstream.BuildElementsFromStreamResult(ctx, res, sqo.TagProjection)
- })
- }
-}
diff --git a/banyand/stream/block.go b/banyand/stream/block.go
index f51fcdefa..108fdb0d4 100644
--- a/banyand/stream/block.go
+++ b/banyand/stream/block.go
@@ -22,10 +22,7 @@ import (
"fmt"
"sort"
- "golang.org/x/exp/slices"
-
"github.com/apache/skywalking-banyandb/api/common"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
pkgbytes "github.com/apache/skywalking-banyandb/pkg/bytes"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/encoding"
@@ -478,101 +475,6 @@ func (bc *blockCursor) init(p *part, bm *blockMetadata, opts queryOptions) {
bc.elementFilter = opts.elementFilter
}
-func (bc *blockCursor) copyAllTo(r *model.StreamResult, desc bool) {
- start, end := 0, bc.idx+1
- if !desc {
- start, end = bc.idx, len(bc.timestamps)
- }
- if end <= start {
- return
- }
-
- r.Timestamps = append(r.Timestamps, bc.timestamps[start:end]...)
- r.ElementIDs = append(r.ElementIDs, bc.elementIDs[start:end]...)
- requiredCapacity := end - start
- r.SIDs = append(r.SIDs, make([]common.SeriesID, requiredCapacity)...)
- for i := range r.SIDs[len(r.SIDs)-requiredCapacity:] {
- r.SIDs[len(r.SIDs)-requiredCapacity+i] = bc.bm.seriesID
- }
-
- if desc {
- slices.Reverse(r.Timestamps)
- slices.Reverse(r.ElementIDs)
- }
-
- if len(r.TagFamilies) != len(bc.tagProjection) {
- r.TagFamilies = make([]model.TagFamily, len(bc.tagProjection))
- for i, tp := range bc.tagProjection {
- r.TagFamilies[i] = model.TagFamily{Name: tp.Family, Tags: make([]model.Tag, len(tp.Names))}
- for j, n := range tp.Names {
- r.TagFamilies[i].Tags[j] = model.Tag{Name: n}
- }
- }
- }
-
- for i, cf := range bc.tagFamilies {
- for j, c := range cf.tags {
- values := make([]*modelv1.TagValue, end-start)
- schemaType, hasSchemaType := bc.schemaTagTypes[c.name]
- for k := start; k < end; k++ {
- if len(c.values) > k {
- if hasSchemaType && c.valueType == schemaType {
- values[k-start] = mustDecodeTagValue(c.valueType, c.values[k])
- } else {
- values[k-start] = pbv1.NullTagValue
- }
- } else {
- values[k-start] = pbv1.NullTagValue
- }
- }
- if desc {
- slices.Reverse(values)
- }
- r.TagFamilies[i].Tags[j].Values = append(r.TagFamilies[i].Tags[j].Values, values...)
- }
- }
-}
-
-func (bc *blockCursor) copyTo(r *model.StreamResult) {
- r.Timestamps = append(r.Timestamps, bc.timestamps[bc.idx])
- r.ElementIDs = append(r.ElementIDs, bc.elementIDs[bc.idx])
- r.SIDs = append(r.SIDs, bc.bm.seriesID)
- if len(r.TagFamilies) != len(bc.tagProjection) {
- for _, tp := range bc.tagProjection {
- tf := model.TagFamily{
- Name: tp.Family,
- }
- for _, n := range tp.Names {
- t := model.Tag{
- Name: n,
- }
- tf.Tags = append(tf.Tags, t)
- }
- r.TagFamilies = append(r.TagFamilies, tf)
- }
- }
- if len(bc.tagFamilies) != len(r.TagFamilies) {
- logger.Panicf("unexpected number of tag families: got %d; want %d", len(bc.tagFamilies), len(r.TagFamilies))
- }
- for i, cf := range bc.tagFamilies {
- if len(r.TagFamilies[i].Tags) != len(cf.tags) {
- logger.Panicf("unexpected number of tags: got %d; want %d", len(r.TagFamilies[i].Tags), len(bc.tagProjection[i].Names))
- }
- for i2, c := range cf.tags {
- schemaType, hasSchemaType := bc.schemaTagTypes[c.name]
- if len(c.values) > bc.idx {
- if hasSchemaType && c.valueType == schemaType {
- r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, mustDecodeTagValue(c.valueType, c.values[bc.idx]))
- } else {
- r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, pbv1.NullTagValue)
- }
- } else {
- r.TagFamilies[i].Tags[i2].Values = append(r.TagFamilies[i].Tags[i2].Values, pbv1.NullTagValue)
- }
- }
- }
-}
-
func (bc *blockCursor) loadData(tmpBlock *block) bool {
tmpBlock.reset()
bc.bm.tagProjection = bc.tagProjection
diff --git a/banyand/stream/query.go b/banyand/stream/query.go
index 5df920d54..a92b4c4ba 100644
--- a/banyand/stream/query.go
+++ b/banyand/stream/query.go
@@ -18,103 +18,19 @@
package stream
import (
- "context"
- "fmt"
- "time"
-
- "github.com/pkg/errors"
+ "errors"
"github.com/apache/skywalking-banyandb/api/common"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/index/posting"
- "github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
- itersort "github.com/apache/skywalking-banyandb/pkg/iter/sort"
"github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/panicdiag"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/pool"
- logicalstream "github.com/apache/skywalking-banyandb/pkg/query/logical/stream"
"github.com/apache/skywalking-banyandb/pkg/query/model"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-var streamQueryResultTracker = pool.RegisterTracker("stream.queryResult")
-
-const checkDoneEvery = 128
-
-func (s *stream) Query(ctx context.Context, sqo model.StreamQueryOptions) (sqr model.StreamQueryResult, err error) {
- ctx = panicdiag.WithBreadcrumb(ctx, "start stream query", "stream", map[string]string{
- "group": s.group,
- "stream": sqo.Name,
- })
- if err = validateQueryInput(sqo); err != nil {
- return nil, err
- }
-
- tsdb, err := s.getTSDB()
- if err != nil {
- return nil, err
- }
-
- segments, err := tsdb.SelectSegments(*sqo.TimeRange, true)
- if err != nil {
- return nil, err
- }
- if len(segments) < 1 {
- return bypassQueryResultInstance, nil
- }
- ctx = panicdiag.WithBreadcrumb(ctx, "selected stream segments", "stream", map[string]string{
- "group": s.group,
- "segments": fmt.Sprintf("%d", len(segments)),
- })
-
- segmentsNeedRelease := true
- defer func() {
- if !segmentsNeedRelease {
- return
- }
- for i := range segments {
- segments[i].DecRef()
- }
- }()
-
- series := prepareSeriesData(sqo)
-
- schemaTagTypes := make(map[string]pbv1.ValueType)
- for _, tf := range s.schema.GetTagFamilies() {
- for _, tag := range tf.GetTags() {
- vt := pbv1.TagValueSpecToValueType(tag.GetType())
- if vt != pbv1.ValueTypeUnknown {
- schemaTagTypes[tag.GetName()] = vt
- }
- }
- }
-
- qo := prepareQueryOptions(sqo, schemaTagTypes)
- tr := index.NewIntRangeOpts(qo.minTimestamp, qo.maxTimestamp, true, true)
-
- if sqo.Order == nil || sqo.Order.Index == nil {
- sqr = s.executeTimeSeriesQuery(segments, series, qo, &tr)
- segmentsNeedRelease = false
- return sqr, nil
- }
-
- ctx = panicdiag.WithBreadcrumb(ctx, "execute stream indexed query", "stream", map[string]string{
- "group": s.group,
- "stream": sqo.Name,
- })
- sqr, err = s.executeIndexedQuery(ctx, segments, series, sqo, schemaTagTypes, &tr)
- if err != nil {
- return nil, err
- }
- segmentsNeedRelease = false
- return sqr, nil
-}
-
func validateQueryInput(sqo model.StreamQueryOptions) error {
if sqo.TimeRange == nil || len(sqo.Entities) < 1 {
return errors.New("invalid query options: timeRange and series are required")
@@ -161,141 +77,6 @@ func prepareQueryOptions(sqo model.StreamQueryOptions, schemaTagTypes map[string
}
}
-func (s *stream) executeTimeSeriesQuery(
- segments []storage.Segment[*tsTable, option],
- series []*pbv1.Series,
- qo queryOptions,
- tr *index.RangeOpts,
-) model.StreamQueryResult {
- result := &tsResult{
- segments: segments,
- series: series,
- qo: qo,
- sm: s,
- pm: s.pm,
- l: s.l,
- tr: tr,
- }
-
- // Determine ascending order
- if qo.Order == nil {
- result.asc = true
- } else if qo.Order.Sort == modelv1.Sort_SORT_ASC || qo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
- result.asc = true
- }
-
- streamQueryResultTracker.Acquire(result)
- return result
-}
-
-func (s *stream) executeIndexedQuery(
- ctx context.Context,
- segments []storage.Segment[*tsTable, option],
- series []*pbv1.Series,
- sqo model.StreamQueryOptions,
- schemaTagTypes map[string]pbv1.ValueType,
- tr *index.RangeOpts,
-) (model.StreamQueryResult, error) {
- result, seriesFilter, resultTS, err := s.processSegmentsAndBuildFilters(ctx, segments, series, sqo, schemaTagTypes, tr)
- if err != nil {
- return nil, err
- }
-
- if seriesFilter.IsEmpty() {
- result.Release()
- return nil, nil
- }
-
- // Update time range if needed
- sids := seriesFilter.ToSlice()
- startTS := sqo.TimeRange.Start.UnixNano()
- endTS := sqo.TimeRange.End.UnixNano()
- minTS, maxTS := updateTimeRange(resultTS, startTS, endTS)
- if minTS > startTS || maxTS < endTS {
- newTR := timestamp.NewTimeRange(time.Unix(0, minTS), time.Unix(0, maxTS), sqo.TimeRange.IncludeStart, sqo.TimeRange.IncludeEnd)
- sqo.TimeRange = &newTR
- }
-
- // Perform index-based sorting
- if result.sortingIter, err = s.indexSort(ctx, sqo, result.tabs, sids); err != nil {
- return nil, err
- }
-
- // Set ascending flag
- if sqo.Order.Sort == modelv1.Sort_SORT_ASC || sqo.Order.Sort == modelv1.Sort_SORT_UNSPECIFIED {
- result.asc = true
- }
-
- streamQueryResultTracker.Acquire(&result)
- return &result, nil
-}
-
-func (s *stream) processSegmentsAndBuildFilters(
- ctx context.Context,
- segments []storage.Segment[*tsTable, option],
- series []*pbv1.Series,
- sqo model.StreamQueryOptions,
- schemaTagTypes map[string]pbv1.ValueType,
- tr *index.RangeOpts,
-) (idxResult, posting.List, posting.List, error) {
- var result idxResult
- result.pm = s.pm
- result.segments = segments
- result.sm = s
- result.qo = queryOptions{
- StreamQueryOptions: sqo,
- schemaTagTypes: schemaTagTypes,
- seriesToEntity: make(map[common.SeriesID][]*modelv1.TagValue),
- }
-
- seriesFilter := roaring.NewPostingList()
- var resultTS posting.List
- var sl pbv1.SeriesList
- var err error
-
- for i := range result.segments {
- sl, err = result.segments[i].Lookup(ctx, series)
- if err != nil {
- return result, nil, nil, err
- }
-
- var filter, filterTS posting.List
- tables, _ := segments[i].Tables()
- if filter, filterTS, err = indexSearch(ctx, sqo, tables, sl.ToList().ToSlice(), tr); err != nil {
- return result, nil, nil, err
- }
-
- if filter != nil && filter.IsEmpty() {
- continue
- }
-
- if result.qo.elementFilter == nil {
- result.qo.elementFilter = filter
- resultTS = filterTS
- } else {
- if err = result.qo.elementFilter.Union(filter); err != nil {
- return result, nil, nil, err
- }
- if err = resultTS.Union(filterTS); err != nil {
- return result, nil, nil, err
- }
- }
-
- for j := range sl {
- if seriesFilter.Contains(uint64(sl[j].ID)) {
- continue
- }
- seriesFilter.Insert(uint64(sl[j].ID))
- result.qo.seriesToEntity[sl[j].ID] = sl[j].EntityValues
- }
-
- tables, _ = result.segments[i].Tables()
- result.tabs = append(result.tabs, tables...)
- }
-
- return result, seriesFilter, resultTS, nil
-}
-
type queryOptions struct {
elementFilter posting.List
seriesToEntity map[common.SeriesID][]*modelv1.TagValue
@@ -326,75 +107,6 @@ func (qo *queryOptions) copyFrom(other *queryOptions) {
qo.maxTimestamp = other.maxTimestamp
}
-func indexSearch(ctx context.Context, sqo model.StreamQueryOptions,
- tabs []*tsTable, seriesList []uint64, tr *index.RangeOpts,
-) (posting.List, posting.List, error) {
- if sqo.InvertedFilter == nil || sqo.InvertedFilter == logicalstream.ENode {
- return nil, nil, nil
- }
- result, resultTS := roaring.NewPostingList(), roaring.NewPostingList()
- for _, tw := range tabs {
- index := tw.Index()
- pl, plTS, err := index.Search(ctx, seriesList, sqo.InvertedFilter, tr)
- if err != nil {
- return nil, nil, err
- }
- if pl == nil || pl.IsEmpty() {
- continue
- }
- if err := result.Union(pl); err != nil {
- return nil, nil, err
- }
- if plTS == nil || plTS.IsEmpty() {
- continue
- }
- if err := resultTS.Union(plTS); err != nil {
- return nil, nil, err
- }
- }
- return result, resultTS, nil
-}
-
-func (s *stream) indexSort(ctx context.Context, sqo model.StreamQueryOptions, tabs []*tsTable,
- sids []uint64,
-) (itersort.Iterator[*index.DocumentResult], error) {
- if sqo.Order == nil || sqo.Order.Index == nil {
- return nil, nil
- }
- seriesList := make([]common.SeriesID, len(sids))
- for i := range sids {
- seriesList[i] = common.SeriesID(sids[i])
- }
- iters, err := s.buildItersByIndex(ctx, tabs, seriesList, sqo)
- if err != nil {
- return nil, err
- }
- desc := sqo.Order != nil && sqo.Order.Sort == modelv1.Sort_SORT_DESC
- return itersort.NewItemIter[*index.DocumentResult](iters, desc), nil
-}
-
-func (s *stream) buildItersByIndex(ctx context.Context, tables []*tsTable,
- sids []common.SeriesID, sqo model.StreamQueryOptions,
-) (iters []itersort.Iterator[*index.DocumentResult], err error) {
- indexRuleForSorting := sqo.Order.Index
- if len(indexRuleForSorting.Tags) != 1 {
- return nil, fmt.Errorf("only support one tag for sorting, but got %d", len(indexRuleForSorting.Tags))
- }
- for _, tw := range tables {
- var iter index.FieldIterator[*index.DocumentResult]
- fieldKey := index.FieldKey{
- IndexRuleID: indexRuleForSorting.GetMetadata().GetId(),
- Analyzer: indexRuleForSorting.GetAnalyzer(),
- }
- iter, err = tw.Index().Sort(ctx, sids, fieldKey, sqo.Order.Sort, sqo.TimeRange, sqo.MaxElementSize)
- if err != nil {
- return nil, err
- }
- iters = append(iters, iter)
- }
- return
-}
-
func mustEncodeTagValue(name string, tagType databasev1.TagType, tagValue *modelv1.TagValue, num int) [][]byte {
values := make([][]byte, num)
tv := encodeTagValue(name, tagType, tagValue)
@@ -503,3 +215,37 @@ func updateTimeRange(filterTS posting.List, minTimestamp, maxTimestamp int64) (i
}
return minTimestamp, maxTimestamp
}
+
+func loadBlockCursor(bc *blockCursor, tmpBlock *block, qo queryOptions, is indexSchema) bool {
+ tmpBlock.reset()
+ if !bc.loadData(tmpBlock) {
+ releaseBlockCursor(bc)
+ return false
+ }
+ entityValues := qo.seriesToEntity[bc.bm.seriesID]
+
+ tagFamilyMap := make(map[string]int)
+ for idx, tagFamily := range bc.tagFamilies {
+ tagFamilyMap[tagFamily.name] = idx + 1
+ }
+ for _, tagFamilyProj := range bc.tagProjection {
+ for j, tagProj := range tagFamilyProj.Names {
+ tagSpec := is.tagMap[tagProj]
+ if tagSpec == nil {
+ continue
+ }
+ entityPos := is.indexRuleLocators.EntitySet[tagProj]
+ if entityPos == 0 {
+ continue
+ }
+ tagFamilyPos := tagFamilyMap[tagFamilyProj.Family]
+ valueType := pbv1.MustTagValueToValueType(entityValues[entityPos-1])
+ bc.tagFamilies[tagFamilyPos-1].tags[j] = tag{
+ name: tagProj,
+ values: mustEncodeTagValue(tagProj, tagSpec.GetType(), entityValues[entityPos-1], len(bc.timestamps)),
+ valueType: valueType,
+ }
+ }
+ }
+ return true
+}
diff --git a/banyand/stream/query_by_idx.go b/banyand/stream/query_by_idx.go
deleted file mode 100644
index f39b19f3a..000000000
--- a/banyand/stream/query_by_idx.go
+++ /dev/null
@@ -1,366 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package stream
-
-import (
- "context"
- "fmt"
- "sort"
-
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/internal/storage"
- "github.com/apache/skywalking-banyandb/banyand/protector"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
- itersort "github.com/apache/skywalking-banyandb/pkg/iter/sort"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/query"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
-)
-
-type idxResult struct {
- sortingIter itersort.Iterator[*index.DocumentResult]
- sm *stream
- pm protector.Memory
- tabs []*tsTable
- elementIDsSorted []uint64
- data []*blockCursor
- snapshots []*snapshot
- segments []storage.Segment[*tsTable, option]
- qo queryOptions
- loaded bool
- asc bool
-}
-
-func (qr *idxResult) Pull(ctx context.Context) *model.StreamResult {
- if !qr.loaded {
- qr.elementIDsSorted = make([]uint64, 0, queryCapacity(qr.qo.MaxElementSize))
- return qr.loadSortingData(ctx)
- }
- if v := qr.nextValue(); v != nil {
- return v
- }
- qr.loaded = false
- return qr.loadSortingData(ctx)
-}
-
-func (qr *idxResult) scanParts(ctx context.Context, qo queryOptions) error {
- var parts []*part
- var n int
- for i := range qr.tabs {
- s := qr.tabs[i].currentSnapshot()
- if s == nil {
- continue
- }
- parts, n = s.getParts(parts, qo.minTimestamp, qo.maxTimestamp)
- if n < 1 {
- s.decRef()
- continue
- }
- qr.snapshots = append(qr.snapshots, s)
- }
- bma := generateBlockMetadataArray()
- defer releaseBlockMetadataArray(bma)
- defFn := startBlockScanSpan(ctx, len(qo.sortedSids), parts, qr)
- defer defFn()
- ti := generateTstIter()
- defer releaseTstIter(ti)
- sids := qo.sortedSids
- ti.init(bma, parts, sids, qo.minTimestamp, qo.maxTimestamp, qo.SkippingFilter)
- if ti.Error() != nil {
- return fmt.Errorf("cannot init tstIter: %w", ti.Error())
- }
- var hit int
- var totalBlockBytes uint64
- quota := qr.pm.AvailableBytes()
- for ti.nextBlock() {
- if hit%checkDoneEvery == 0 {
- select {
- case <-ctx.Done():
- return errors.WithMessagef(ctx.Err(), "interrupt: scanned %d blocks, remained %d/%d parts to scan", len(qr.data), len(ti.piHeap), len(ti.piPool))
- default:
- }
- }
- hit++
- p := ti.piHeap[0]
- if chargeErr := query.Charge(ctx, p.curBlock.uncompressedSizeBytes); chargeErr != nil {
- return chargeErr
- }
- bc := generateBlockCursor()
- bc.init(p.p, p.curBlock, qo)
- qr.data = append(qr.data, bc)
- totalBlockBytes += bc.bm.uncompressedSizeBytes
- if quota >= 0 && totalBlockBytes > uint64(quota) {
- return fmt.Errorf("parts scan quota exceeded: used %d bytes, quota is %d bytes", totalBlockBytes, quota)
- }
- }
- if ti.Error() != nil {
- return fmt.Errorf("cannot iterate tstIter: %w", ti.Error())
- }
- if err := qr.pm.AcquireResource(ctx, totalBlockBytes); err != nil {
- return fmt.Errorf("cannot acquire resource: %w", err)
- }
- return nil
-}
-
-func (qr *idxResult) load(ctx context.Context, qo queryOptions) *model.StreamResult {
- if qr.loaded {
- qr.nextValue()
- }
- if err := qr.scanParts(ctx, qo); err != nil {
- return &model.StreamResult{
- Error: err,
- }
- }
- if len(qr.data) == 0 {
- return nil
- }
-
- cursorChan := make(chan int, len(qr.data))
- is := qr.sm.indexSchema.Load().(indexSchema)
- idxBlockLogger := logger.GetLogger("stream-idx-block-loader")
- for i := 0; i < len(qr.data); i++ {
- idx := i
- go func() {
- defer func() {
- if r := recover(); r != nil {
- idxBlockLogger.Error().Interface("panic", r).Msg("panic in parallel block loader")
- cursorChan <- idx
- }
- }()
- select {
- case <-ctx.Done():
- releaseBlockCursor(qr.data[idx])
- cursorChan <- idx
- return
- default:
- }
- if qr.sm.schema.GetEntity() == nil || len(qr.sm.schema.GetEntity().GetTagNames()) == 0 {
- cursorChan <- -1
- return
- }
- tmpBlock := generateBlock()
- defer releaseBlock(tmpBlock)
- if loadBlockCursor(qr.data[idx], tmpBlock, qo, is) {
- cursorChan <- -1
- return
- }
- cursorChan <- idx
- }()
- }
-
- blankCursorList := []int{}
- for completed := 0; completed < len(qr.data); completed++ {
- result := <-cursorChan
- if result != -1 {
- blankCursorList = append(blankCursorList, result)
- }
- }
- select {
- case <-ctx.Done():
- return &model.StreamResult{
- Error: errors.WithMessagef(ctx.Err(), "interrupt: blank/total=%d/%d", len(blankCursorList), len(qr.data)),
- }
- default:
- }
- sort.Slice(blankCursorList, func(i, j int) bool {
- return blankCursorList[i] > blankCursorList[j]
- })
- for _, index := range blankCursorList {
- qr.data = append(qr.data[:index], qr.data[index+1:]...)
- }
- qr.loaded = true
- return qr.nextValue()
-}
-
-func (qr *idxResult) nextValue() *model.StreamResult {
- if len(qr.data) == 0 {
- return nil
- }
- return qr.mergeByTagValue()
-}
-
-func (qr *idxResult) loadSortingData(ctx context.Context) *model.StreamResult {
- var qo queryOptions
- qo.StreamQueryOptions = qr.qo.StreamQueryOptions
- qo.elementFilter = roaring.NewPostingList()
- qo.seriesToEntity = qr.qo.seriesToEntity
- qo.schemaTagTypes = qr.qo.schemaTagTypes
- qr.elementIDsSorted = qr.elementIDsSorted[:0]
- count, searchedSize := 1, 0
- seenElementIDs := make(map[uint64]bool)
- tracer := query.GetTracer(ctx)
- if tracer != nil {
- span, _ := tracer.StartSpan(ctx, "load-sorting-data")
- span.Tagf("max_element_size", "%d", qo.MaxElementSize)
- if qr.qo.elementFilter != nil {
- span.Tag("filter_size", fmt.Sprintf("%d", qr.qo.elementFilter.Len()))
- }
- defer func() {
- span.Tagf("searched_size", "%d", searchedSize)
- span.Tagf("count", "%d", count)
- span.Stop()
- }()
- }
- for ; qr.sortingIter.Next(); count++ {
- searchedSize++
- val := qr.sortingIter.Val()
- if qr.qo.elementFilter != nil && !qr.qo.elementFilter.Contains(val.DocID) {
- count--
- continue
- }
- // Deduplicate element IDs
- if seenElementIDs[val.DocID] {
- count--
- continue
- }
- if chargeErr := query.Charge(ctx, 64); chargeErr != nil {
- return &model.StreamResult{Error: chargeErr}
- }
- seenElementIDs[val.DocID] = true
- qo.elementFilter.Insert(val.DocID)
- if val.Timestamp > qo.maxTimestamp {
- qo.maxTimestamp = val.Timestamp
- }
- if val.Timestamp < qo.minTimestamp || qo.minTimestamp == 0 {
- qo.minTimestamp = val.Timestamp
- }
- qr.elementIDsSorted = append(qr.elementIDsSorted, val.DocID)
-
- // Insertion sort
- insertPos, found := -1, false
- for i, sid := range qo.sortedSids {
- if val.SeriesID == sid {
- found = true
- break
- }
- if val.SeriesID < sid {
- insertPos = i
- break
- }
- }
-
- if !found {
- if insertPos == -1 {
- qo.sortedSids = append(qo.sortedSids, val.SeriesID)
- } else {
- qo.sortedSids = append(qo.sortedSids[:insertPos], append([]common.SeriesID{val.SeriesID}, qo.sortedSids[insertPos:]...)...)
- }
- }
- if count >= qo.MaxElementSize {
- break
- }
- }
- if qo.elementFilter.IsEmpty() {
- return nil
- }
- return qr.load(ctx, qo)
-}
-
-func (qr *idxResult) releaseParts() {
- qr.releaseBlockCursor()
- for i := range qr.snapshots {
- qr.snapshots[i].decRef()
- }
- qr.snapshots = qr.snapshots[:0]
-}
-
-func (qr *idxResult) releaseBlockCursor() {
- for i, v := range qr.data {
- releaseBlockCursor(v)
- qr.data[i] = nil
- }
- qr.data = qr.data[:0]
-}
-
-func (qr *idxResult) Release() {
- streamQueryResultTracker.Release(qr)
- qr.releaseParts()
- for i := range qr.segments {
- qr.segments[i].DecRef()
- }
-}
-
-func (qr *idxResult) mergeByTagValue() *model.StreamResult {
- defer qr.releaseBlockCursor()
- tmp := &model.StreamResult{}
- prevIdx := 0
- elementIDToIdx := make(map[uint64]int)
- for _, data := range qr.data {
- data.copyAllTo(tmp, false)
- var idx int
- for idx = prevIdx; idx < len(tmp.Timestamps); idx++ {
- // Keep the first occurrence of each element ID
- if _, exists := elementIDToIdx[tmp.ElementIDs[idx]]; !exists {
- elementIDToIdx[tmp.ElementIDs[idx]] = idx
- }
- }
- prevIdx = idx
- }
-
- r := &model.StreamResult{
- TagFamilies: []model.TagFamily{},
- }
- for _, tagFamily := range tmp.TagFamilies {
- tf := model.TagFamily{
- Name: tagFamily.Name,
- Tags: []model.Tag{},
- }
- for _, tag := range tagFamily.Tags {
- t := model.Tag{
- Name: tag.Name,
- Values: []*modelv1.TagValue{},
- }
- tf.Tags = append(tf.Tags, t)
- }
- r.TagFamilies = append(r.TagFamilies, tf)
- }
- seenElementIDs := make(map[uint64]bool)
- for _, id := range qr.elementIDsSorted {
- // Deduplicate: only process each element ID once
- if seenElementIDs[id] {
- continue
- }
- seenElementIDs[id] = true
- idx, ok := elementIDToIdx[id]
- if !ok {
- continue
- }
- r.Timestamps = append(r.Timestamps, tmp.Timestamps[idx])
- r.ElementIDs = append(r.ElementIDs, tmp.ElementIDs[idx])
- for i := 0; i < len(r.TagFamilies); i++ {
- for j := 0; j < len(r.TagFamilies[i].Tags); j++ {
- r.TagFamilies[i].Tags[j].Values = append(r.TagFamilies[i].Tags[j].Values, tmp.TagFamilies[i].Tags[j].Values[idx])
- }
- }
- }
- return r
-}
-
-var bypassQueryResultInstance = &bypassQueryResult{}
-
-type bypassQueryResult struct{}
-
-func (bypassQueryResult) Pull(context.Context) *model.StreamResult {
- return nil
-}
-
-func (bypassQueryResult) Release() {}
diff --git a/banyand/stream/query_by_idx_test.go b/banyand/stream/query_by_idx_test.go
deleted file mode 100644
index 05af46e7f..000000000
--- a/banyand/stream/query_by_idx_test.go
+++ /dev/null
@@ -1,325 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package stream
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/api/common"
- itest "github.com/apache/skywalking-banyandb/banyand/internal/test"
- "github.com/apache/skywalking-banyandb/banyand/protector"
- "github.com/apache/skywalking-banyandb/pkg/fs"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-func TestQueryResult_QuotaExceeded(t *testing.T) {
- type testCtx struct {
- name string
- esList []*elements
- sids []common.SeriesID
- want []blockMetadata
- minTimestamp int64
- maxTimestamp int64
- expectQuotaExceeded bool
- }
-
- tests := []testCtx{
- {
- name: "TestQuotaNotExceeded_ExpectSuccess",
- esList: []*elements{esTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- expectQuotaExceeded: false,
- want: []blockMetadata{
- {seriesID: 1, count: 1, uncompressedSizeBytes: 859},
- {seriesID: 2, count: 1, uncompressedSizeBytes: 47},
- {seriesID: 3, count: 1, uncompressedSizeBytes: 16},
- },
- },
- {
- name: "TestQuotaExceeded_ExpectError",
- esList: []*elements{esTS1},
- sids: []common.SeriesID{1, 2, 3},
- minTimestamp: 1,
- maxTimestamp: 1,
- expectQuotaExceeded: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- // Initialize a tstIter object.
- tmpPath, defFn := test.Space(require.New(t))
- fileSystem := fs.NewLocalFileSystem()
- defer defFn()
- tst, err := newTSTable(fileSystem, tmpPath, common.Position{},
- logger.GetLogger("test"), timestamp.TimeRange{}, option{flushTimeout: 0, mergePolicy: newDefaultMergePolicyForTesting(), protector: protector.Nop{}}, nil)
- require.NoError(t, err)
- for _, es := range tt.esList {
- tst.mustAddElements(es)
- time.Sleep(100 * time.Millisecond)
- }
- // wait until every introduced mem part has been flushed to disk;
- // inspecting only snapshot.creator races with concurrent introductions
- // (a flush of part N can land while part N+1 is still a mem part).
- if len(tt.esList) > 0 {
- require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
- "mem parts not flushed to disk in time")
- tst.Close()
- }
-
- // reopen the table
- tst, err = newTSTable(fileSystem, tmpPath, common.Position{},
- logger.GetLogger("test"), timestamp.TimeRange{}, option{
- flushTimeout: defaultFlushTimeout, mergePolicy: newDefaultMergePolicyForTesting(),
- protector: protector.Nop{},
- }, nil)
- require.NoError(t, err)
-
- defer tst.Close()
- qr := &idxResult{
- pm: &itest.MockMemoryProtector{ExpectQuotaExceeded: tt.expectQuotaExceeded},
- tabs: []*tsTable{tst},
- }
- qo := queryOptions{
- minTimestamp: tt.minTimestamp,
- maxTimestamp: tt.maxTimestamp,
- sortedSids: tt.sids,
- }
- err = qr.scanParts(context.TODO(), qo)
- if tt.expectQuotaExceeded {
- require.Error(t, err)
- require.Contains(t, err.Error(), "quota exceeded", "expected quota to be exceeded but got: %v", err)
- return
- }
- require.NoError(t, err)
- var got []blockMetadata
- for _, data := range qr.data {
- got = append(got, data.bm)
- }
- if diff := cmp.Diff(got, tt.want,
- cmpopts.IgnoreFields(blockMetadata{}, "timestamps"),
- cmpopts.IgnoreFields(blockMetadata{}, "elementIDs"),
- cmpopts.IgnoreFields(blockMetadata{}, "tagFamilies"),
- cmpopts.IgnoreFields(blockMetadata{}, "tagType"),
- cmp.AllowUnexported(blockMetadata{}),
- ); diff != "" {
- t.Errorf("Unexpected blockMetadata (-got +want):\n%s", diff)
- }
- })
- }
-}
-
-func TestIdxResult_MergeByTagValue_Deduplication(t *testing.T) {
- tests := []struct {
- name string
- data []*blockCursor
- elementIDsSorted []uint64
- wantElementIDs []uint64
- wantLen int
- }{
- {
- name: "duplicate element IDs in block cursors should be deduplicated",
- data: []*blockCursor{
- {
- timestamps: []int64{10, 20},
- elementIDs: []uint64{100, 200},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{15, 25},
- elementIDs: []uint64{100, 300}, // 100 is duplicate
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup"), []byte("value3")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- elementIDsSorted: []uint64{100, 200, 300},
- wantElementIDs: []uint64{100, 200, 300},
- wantLen: 3,
- },
- {
- name: "duplicate element IDs in elementIDsSorted should be deduplicated",
- data: []*blockCursor{
- {
- timestamps: []int64{10, 20, 30},
- elementIDs: []uint64{100, 200, 300},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2"), []byte("value3")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- elementIDsSorted: []uint64{100, 200, 100, 300}, // 100 appears twice
- wantElementIDs: []uint64{100, 200, 300}, // Should be deduplicated
- wantLen: 3,
- },
- {
- name: "duplicate element IDs in both data and elementIDsSorted",
- data: []*blockCursor{
- {
- timestamps: []int64{10, 20},
- elementIDs: []uint64{100, 200},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{15},
- elementIDs: []uint64{100}, // duplicate in data
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- elementIDsSorted: []uint64{100, 200, 100}, // 100 appears twice in sorted list
- wantElementIDs: []uint64{100, 200}, // Should be deduplicated
- wantLen: 2,
- },
- {
- name: "all duplicates should result in single element",
- data: []*blockCursor{
- {
- timestamps: []int64{10},
- elementIDs: []uint64{100},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{20},
- elementIDs: []uint64{100}, // duplicate
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- elementIDsSorted: []uint64{100, 100, 100}, // all duplicates
- wantElementIDs: []uint64{100}, // Should result in single element
- wantLen: 1,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- qr := &idxResult{
- data: tt.data,
- elementIDsSorted: tt.elementIDsSorted,
- }
- result := qr.mergeByTagValue()
-
- require.Equal(t, tt.wantLen, result.Len(), "unexpected length")
- require.Equal(t, tt.wantElementIDs, result.ElementIDs, "unexpected element IDs - duplicates should be removed")
- // Verify no duplicate element IDs
- seen := make(map[uint64]bool)
- for _, id := range result.ElementIDs {
- require.False(t, seen[id], "duplicate element ID found: %d", id)
- seen[id] = true
- }
- })
- }
-}
diff --git a/banyand/stream/query_by_ts.go b/banyand/stream/query_by_ts.go
deleted file mode 100644
index f47958eac..000000000
--- a/banyand/stream/query_by_ts.go
+++ /dev/null
@@ -1,304 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package stream
-
-import (
- "container/heap"
- "context"
- "sync"
-
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/banyand/internal/storage"
- "github.com/apache/skywalking-banyandb/banyand/protector"
- "github.com/apache/skywalking-banyandb/pkg/cgroups"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/pool"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-var _ model.StreamQueryResult = (*tsResult)(nil)
-
-type tsResult struct {
- sm *stream
- pm protector.Memory
- l *logger.Logger
- ts *blockScanner
- tr *index.RangeOpts
- segments []storage.Segment[*tsTable, option]
- series []*pbv1.Series
- shards []*model.StreamResult
- qo queryOptions
- asc bool
-}
-
-func (t *tsResult) Pull(ctx context.Context) (r *model.StreamResult) {
- if len(t.segments) == 0 && t.ts == nil {
- return nil
- }
- var err error
- if r, err = t.scan(ctx); err != nil {
- return &model.StreamResult{Error: err}
- }
- return r
-}
-
-func (t *tsResult) scan(ctx context.Context) (*model.StreamResult, error) {
- if t.ts != nil {
- return t.runTabScanner(ctx)
- }
- for len(t.segments) > 0 {
- var segment storage.Segment[*tsTable, option]
- if t.asc {
- segment = t.segments[len(t.segments)-1]
- t.segments = t.segments[:len(t.segments)-1]
- } else {
- segment = t.segments[0]
- t.segments = t.segments[1:]
- }
- qo, err := searchSeries(ctx, t.qo, segment, t.series)
- if err != nil {
- return nil, err
- }
- ts, err := getBlockScanner(ctx, segment, qo, t.l, t.pm, t.tr)
- if err != nil {
- return nil, err
- }
- if ts == nil {
- continue
- }
- t.ts = ts
- return t.runTabScanner(ctx)
- }
- return nil, nil
-}
-
-func (t *tsResult) runTabScanner(ctx context.Context) (*model.StreamResult, error) {
- workerSize := cgroups.CPUs()
- var workerWg sync.WaitGroup
- batchCh := make(chan *blockScanResultBatch, workerSize)
- workerWg.Add(workerSize)
- if t.shards == nil {
- t.shards = make([]*model.StreamResult, workerSize)
- for i := range t.shards {
- t.shards[i] = model.NewStreamResult(t.qo.MaxElementSize, t.asc)
- }
- } else {
- for i := range t.shards {
- t.shards[i].Reset()
- }
- }
- is := t.sm.indexSchema.Load().(indexSchema)
- for i := range workerSize {
- workerID := i
- run.Go(ctx, "stream-ts-worker", t.l, func(_ context.Context) {
- defer workerWg.Done()
- tmpBlock := generateBlock()
- defer releaseBlock(tmpBlock)
- blockHeap := generateBlockCursorHeap(t.asc)
- defer releaseBlockCursorHeap(blockHeap)
- tmpResult := model.NewStreamResult(t.qo.MaxElementSize, t.asc)
- for batch := range batchCh {
- if batch.err != nil {
- t.shards[workerID].Error = batch.err
- releaseBlockScanResultBatch(batch)
- continue
- }
- for _, bs := range batch.bss {
- bc := generateBlockCursor()
- bc.init(bs.p, &bs.bm, bs.qo)
- if loadBlockCursor(bc, tmpBlock, bs.qo, is) {
- if !t.asc {
- bc.idx = len(bc.timestamps) - 1
- }
- blockHeap.Push(bc)
- }
- }
- heap.Init(blockHeap)
- result := blockHeap.merge(t.qo.MaxElementSize)
- t.shards[workerID].CopyFrom(tmpResult, result)
- blockHeap.reset()
- releaseBlockScanResultBatch(batch)
- }
- })
- }
- t.ts.scan(ctx, batchCh)
- close(batchCh)
- workerWg.Wait()
- if len(t.ts.parts) == 0 {
- t.ts.close()
- t.ts = nil
- }
- var err error
- for i := range t.shards {
- if t.shards[i].Error != nil {
- err = multierr.Append(err, t.shards[i].Error)
- }
- }
- if err != nil {
- return nil, err
- }
- return model.MergeStreamResults(t.shards, t.qo.MaxElementSize, t.asc), nil
-}
-
-func loadBlockCursor(bc *blockCursor, tmpBlock *block, qo queryOptions, is indexSchema) bool {
- tmpBlock.reset()
- if !bc.loadData(tmpBlock) {
- releaseBlockCursor(bc)
- return false
- }
- entityValues := qo.seriesToEntity[bc.bm.seriesID]
-
- tagFamilyMap := make(map[string]int)
- for idx, tagFamily := range bc.tagFamilies {
- tagFamilyMap[tagFamily.name] = idx + 1
- }
- for _, tagFamilyProj := range bc.tagProjection {
- for j, tagProj := range tagFamilyProj.Names {
- tagSpec := is.tagMap[tagProj]
- if tagSpec == nil {
- continue
- }
- entityPos := is.indexRuleLocators.EntitySet[tagProj]
- if entityPos == 0 {
- continue
- }
- tagFamilyPos := tagFamilyMap[tagFamilyProj.Family]
- valueType := pbv1.MustTagValueToValueType(entityValues[entityPos-1])
- bc.tagFamilies[tagFamilyPos-1].tags[j] = tag{
- name: tagProj,
- values: mustEncodeTagValue(tagProj, tagSpec.GetType(), entityValues[entityPos-1], len(bc.timestamps)),
- valueType: valueType,
- }
- }
- }
- return true
-}
-
-func (t *tsResult) Release() {
- streamQueryResultTracker.Release(t)
- if t.ts != nil {
- t.ts.close()
- }
- for i := range t.segments {
- t.segments[i].DecRef()
- }
-}
-
-type blockCursorHeap struct {
- bcc []*blockCursor
- asc bool
-}
-
-func (bch blockCursorHeap) Len() int {
- return len(bch.bcc)
-}
-
-func (bch blockCursorHeap) Less(i, j int) bool {
- leftIdx, rightIdx := bch.bcc[i].idx, bch.bcc[j].idx
- leftTS := bch.bcc[i].timestamps[leftIdx]
- rightTS := bch.bcc[j].timestamps[rightIdx]
- if bch.asc {
- return leftTS < rightTS
- }
- return leftTS > rightTS
-}
-
-func (bch *blockCursorHeap) Swap(i, j int) {
- bch.bcc[i], bch.bcc[j] = bch.bcc[j], bch.bcc[i]
-}
-
-func (bch *blockCursorHeap) Push(x interface{}) {
- bch.bcc = append(bch.bcc, x.(*blockCursor))
-}
-
-func (bch *blockCursorHeap) Pop() interface{} {
- old := bch.bcc
- n := len(old)
- x := old[n-1]
- bch.bcc = old[0 : n-1]
- releaseBlockCursor(x)
- return x
-}
-
-func (bch *blockCursorHeap) reset() {
- for i := range bch.bcc {
- releaseBlockCursor(bch.bcc[i])
- }
- bch.bcc = bch.bcc[:0]
-}
-
-func (bch *blockCursorHeap) merge(limit int) *model.StreamResult {
- step := -1
- if bch.asc {
- step = 1
- }
- result := &model.StreamResult{}
- seenElementIDs := make(map[uint64]bool)
-
- for bch.Len() > 0 {
- topBC := bch.bcc[0]
- elementID := topBC.elementIDs[topBC.idx]
- if !seenElementIDs[elementID] {
- seenElementIDs[elementID] = true
- topBC.copyTo(result)
- if result.Len() >= limit {
- break
- }
- }
- topBC.idx += step
-
- if bch.asc {
- if topBC.idx >= len(topBC.timestamps) {
- heap.Pop(bch)
- } else {
- heap.Fix(bch, 0)
- }
- } else {
- if topBC.idx < 0 {
- heap.Pop(bch)
- } else {
- heap.Fix(bch, 0)
- }
- }
- }
-
- return result
-}
-
-var blockCursorHeapPool = pool.Register[*blockCursorHeap]("stream-blockCursorHeap")
-
-func generateBlockCursorHeap(asc bool) *blockCursorHeap {
- v := blockCursorHeapPool.Get()
- if v == nil {
- return &blockCursorHeap{
- asc: asc,
- bcc: make([]*blockCursor, 0, blockScannerBatchSize),
- }
- }
- v.asc = asc
- return v
-}
-
-func releaseBlockCursorHeap(bch *blockCursorHeap) {
- bch.reset()
- blockCursorHeapPool.Put(bch)
-}
diff --git a/banyand/stream/query_by_ts_test.go b/banyand/stream/query_by_ts_test.go
deleted file mode 100644
index ed8aa1820..000000000
--- a/banyand/stream/query_by_ts_test.go
+++ /dev/null
@@ -1,266 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package stream
-
-import (
- "container/heap"
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/skywalking-banyandb/api/common"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
-)
-
-func TestBlockCursorHeap_Merge_Deduplication(t *testing.T) {
- tests := []struct {
- name string
- cursors []*blockCursor
- wantElementIDs []uint64
- limit int
- wantLen int
- asc bool
- }{
- {
- name: "duplicate element IDs should be deduplicated",
- cursors: []*blockCursor{
- {
- timestamps: []int64{10, 20},
- elementIDs: []uint64{100, 200},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{15, 25},
- elementIDs: []uint64{100, 300}, // 100 is duplicate
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup"), []byte("value3")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- asc: true,
- limit: 10,
- wantElementIDs: []uint64{100, 200, 300},
- wantLen: 3,
- },
- {
- name: "all duplicates should result in single element",
- cursors: []*blockCursor{
- {
- timestamps: []int64{10},
- elementIDs: []uint64{100},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{20},
- elementIDs: []uint64{100}, // duplicate
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- asc: true,
- limit: 10,
- wantElementIDs: []uint64{100},
- wantLen: 1,
- },
- {
- name: "multiple duplicates across multiple cursors",
- cursors: []*blockCursor{
- {
- timestamps: []int64{10, 20},
- elementIDs: []uint64{100, 200},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{15, 25},
- elementIDs: []uint64{100, 200}, // both duplicates
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup"), []byte("value2_dup")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{30},
- elementIDs: []uint64{300},
- idx: 0,
- bm: blockMetadata{seriesID: common.SeriesID(3)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value3")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- asc: true,
- limit: 10,
- wantElementIDs: []uint64{100, 200, 300},
- wantLen: 3,
- },
- {
- name: "descending order with duplicates",
- cursors: []*blockCursor{
- {
- timestamps: []int64{30, 20},
- elementIDs: []uint64{100, 200},
- idx: 1, // start from last element for descending order
- bm: blockMetadata{seriesID: common.SeriesID(1)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1"), []byte("value2")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- {
- timestamps: []int64{25, 15},
- elementIDs: []uint64{100, 300}, // 100 is duplicate
- idx: 1, // start from last element for descending order
- bm: blockMetadata{seriesID: common.SeriesID(2)},
- tagProjection: []model.TagProjection{
- {Family: "family1", Names: []string{"tag1"}},
- },
- tagFamilies: []tagFamily{
- {
- name: "family1",
- tags: []tag{
- {name: "tag1", values: [][]byte{[]byte("value1_dup"), []byte("value3")}},
- },
- },
- },
- schemaTagTypes: make(map[string]pbv1.ValueType),
- },
- },
- asc: false,
- limit: 10,
- // For descending order: process from highest timestamp to lowest
- // The actual order depends on heap processing, but duplicates are correctly removed
- // Actual result: [200, 100, 300] - all unique, no duplicates
- wantElementIDs: []uint64{200, 100, 300},
- wantLen: 3,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- bch := &blockCursorHeap{
- asc: tt.asc,
- bcc: make([]*blockCursor, 0, len(tt.cursors)),
- }
- for _, bc := range tt.cursors {
- heap.Push(bch, bc)
- }
- heap.Init(bch)
-
- result := bch.merge(tt.limit)
-
- assert.Equal(t, tt.wantLen, result.Len(), "unexpected length")
- assert.Equal(t, tt.wantElementIDs, result.ElementIDs, "unexpected element IDs - duplicates should be removed")
- // Verify no duplicate element IDs
- seen := make(map[uint64]bool)
- for _, id := range result.ElementIDs {
- assert.False(t, seen[id], "duplicate element ID found: %d", id)
- seen[id] = true
- }
- })
- }
-}
diff --git a/banyand/stream/query_capacity.go b/banyand/stream/query_capacity.go
deleted file mode 100644
index e419db601..000000000
--- a/banyand/stream/query_capacity.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with this work for additional
-// information regarding copyright ownership. The ASF licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-package stream
-
-const queryCapacityHint = 1024
-
-func queryCapacity(size int) int {
- if size > queryCapacityHint {
- return queryCapacityHint
- }
- if size < 0 {
- return 0
- }
- return size
-}
diff --git a/banyand/stream/query_capacity_test.go b/banyand/stream/query_capacity_test.go
deleted file mode 100644
index cc41f7191..000000000
--- a/banyand/stream/query_capacity_test.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with this work for additional
-// information regarding copyright ownership. The ASF licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-package stream
-
-import (
- "context"
- "math"
- "testing"
-
- "github.com/stretchr/testify/require"
-
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/query/model"
-)
-
-// The stream index-order path derived MaxElementSize from limit+offset and
-// preallocated elementIDsSorted with that capacity. When the vectorized path
-// declines (order tag omitted from projection), this row path must stay bounded.
-func TestQueryCapacityCapsIndexOrderElementIDs(t *testing.T) {
- if got := queryCapacity(math.MaxInt32); got != queryCapacityHint {
- t.Fatalf("queryCapacity(MaxInt32)=%d, want %d", got, queryCapacityHint)
- }
- if got := queryCapacity(64); got != 64 {
- t.Fatalf("queryCapacity(64)=%d, want 64", got)
- }
- const oversized = 4096
- ids := make([]uint64, 0, queryCapacity(oversized))
- if cap(ids) != queryCapacityHint {
- t.Fatalf("elementIDsSorted capacity %d, want %d", cap(ids), queryCapacityHint)
- }
-}
-
-func TestIndexOrderFallbackElementIDsCapacityBounded(t *testing.T) {
- const oversizedMaxElementSize = 4096
- indexRule := &databasev1.IndexRule{
- Metadata: &commonv1.Metadata{Name: "filter-idx", Id: indexOrderRuleID},
- Tags: []string{"filter-tag"},
- }
- s, tr := buildIndexOrderStream(t)
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: parityEntities(indexOrderSeriesCount),
- // Order by filter-tag but project only entity-tag so the vectorized path
- // declines and the row index-sort path allocates elementIDsSorted.
- TagProjection: []model.TagProjection{{
- Family: "benchmark-family",
- Names: []string{"entity-tag"},
- }},
- Order: &index.OrderBy{Index: indexRule, Sort: modelv1.Sort_SORT_ASC},
- MaxElementSize: oversizedMaxElementSize,
- }
- res, err := s.Query(context.Background(), sqo)
- require.NoError(t, err)
- require.NotNil(t, res)
- defer res.Release()
-
- qr, ok := res.(*idxResult)
- require.True(t, ok, "expected idxResult from indexed-order query")
- _ = qr.Pull(context.Background())
- require.Equal(t, queryCapacityHint, cap(qr.elementIDsSorted),
- "idxResult.elementIDsSorted must stay hint-capped for oversized MaxElementSize")
-}
diff --git a/banyand/stream/query_vectorized_parity_test.go b/banyand/stream/query_vectorized_parity_test.go
index 3fa3f45a1..1e66de2e4 100644
--- a/banyand/stream/query_vectorized_parity_test.go
+++ b/banyand/stream/query_vectorized_parity_test.go
@@ -19,14 +19,18 @@ package stream
import (
"context"
+ "encoding/hex"
"fmt"
"math"
+ "os"
+ "path/filepath"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
@@ -38,7 +42,6 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- logicalstream "github.com/apache/skywalking-banyandb/pkg/query/logical/stream"
"github.com/apache/skywalking-banyandb/pkg/query/model"
"github.com/apache/skywalking-banyandb/pkg/query/vectorized"
vstream "github.com/apache/skywalking-banyandb/pkg/query/vectorized/stream"
@@ -50,8 +53,92 @@ const (
indexOrderRuleID = 1
indexOrderSeriesCount = 4
indexOrderTSCount = 12
+ entityTagName = "entity-tag"
)
+// parityFixture is a golden expectation file under testdata/. The files were
+// captured from the row query path before it was removed (apache/skywalking#13998)
+// and cannot be regenerated; they are the frozen oracle these tests assert against.
+type parityFixture struct {
+ Name string `yaml:"name"`
+ Cases []parityCase `yaml:"cases"`
+}
+
+// parityCase is one query shape within a golden file.
+type parityCase struct {
+ Name string `yaml:"name"`
+ Query string `yaml:"query"`
+ Elements []parityElement `yaml:"elements"`
+ ElementCount int `yaml:"element_count"`
+}
+
+// parityElement is one expected element. TS is a nanosecond OFFSET from the query
+// time range start, because the fixture writer anchors element timestamps at
+// time.Now(); only entity-tag is recorded because the sibling filter-tag is drawn
+// from crypto/rand per run and no golden can pin it.
+type parityElement struct {
+ ID string `yaml:"id"`
+ EntityTag string `yaml:"entity-tag"`
+ TS int64 `yaml:"ts"`
+}
+
+// loadParityGolden reads a golden file from testdata/ and indexes its cases by name.
+func loadParityGolden(t *testing.T, name string) map[string]parityCase {
+ t.Helper()
+ raw, err := os.ReadFile(filepath.Join("testdata", name+".yaml"))
+ require.NoError(t, err)
+ var fixture parityFixture
+ require.NoError(t, yaml.Unmarshal(raw, &fixture))
+ require.Equal(t, name, fixture.Name, "golden file name does not match its content")
+ byName := make(map[string]parityCase, len(fixture.Cases))
+ for _, c := range fixture.Cases {
+ require.Len(t, c.Elements, c.ElementCount, "golden case %q is self-inconsistent", c.Name)
+ byName[c.Name] = c
+ }
+ return byName
+}
+
+// entityTagValue returns the element's entity-tag value.
+func entityTagValue(t *testing.T, e *streamv1.Element) string {
+ t.Helper()
+ for _, family := range e.TagFamilies {
+ for _, tag := range family.Tags {
+ if tag.Key == entityTagName {
+ return tag.Value.GetStr().GetValue()
+ }
+ }
+ }
+ t.Fatalf("element %s carries no %s", e.ElementId, entityTagName)
+ return ""
+}
+
+// assertGoldenParity asserts the vec elements match the golden case: the same
+// element-id set, each with the recorded timestamp offset from trStart and the
+// recorded entity-tag. exact additionally pins the ORDER, which only holds for
+// fixtures with globally unique timestamps (seriesCount=1); multi-series fixtures
+// stamp duplicate timestamps whose relative order is not path-stable, so those
+// cases assert the set plus ts monotonicity instead.
+func assertGoldenParity(t *testing.T, want parityCase, got []*streamv1.Element, trStart time.Time, exact, desc bool) {
+ t.Helper()
+ // A missing map key yields a zero parityCase, which would make an empty-result
+ // assertion pass vacuously; Name is only set when the lookup actually hit.
+ require.NotEmpty(t, want.Name, "golden case not found in its fixture")
+ require.Len(t, got, want.ElementCount, "case %q: element count mismatch", want.Name)
+ gotByID := indexByElementID(t, got)
+ for idx, wantElem := range want.Elements {
+ gotElem, ok := gotByID[wantElem.ID]
+ require.True(t, ok, "case %q: vec path missing golden elementID %s", want.Name, wantElem.ID)
+ require.Equal(t, wantElem.TS, gotElem.Timestamp.AsTime().UnixNano()-trStart.UnixNano(),
+ "case %q: ts offset mismatch for %s", want.Name, wantElem.ID)
+ require.Equal(t, wantElem.EntityTag, entityTagValue(t, gotElem),
+ "case %q: entity-tag mismatch for %s", want.Name, wantElem.ID)
+ if exact {
+ require.Equal(t, wantElem.ID, got[idx].ElementId, "case %q: element order mismatch at %d", want.Name, idx)
+ }
+ }
+ assertMonotonicTS(t, got, desc)
+}
+
// parityEntities builds the per-series entity tag-value lists the fixture writer
// keys on (entity1..entityN), matching the entity index docs writeVecFixture adds.
func parityEntities(seriesCount int) [][]*modelv1.TagValue {
@@ -68,17 +155,17 @@ func parityEntities(seriesCount int) [][]*modelv1.TagValue {
func fullProjection() []model.TagProjection {
return []model.TagProjection{{
Family: "benchmark-family",
- Names: []string{"entity-tag", "filter-tag"},
+ Names: []string{entityTagName, "filter-tag"},
}}
}
// runVecPipeline drives the vec scan through the FULL M4 pipeline
// (merge → distinct → limit) exactly as localIndexScan.ExecuteVectorized does at
// the data-node standalone path, then materializes []*streamv1.Element via the
-// batch egress. limitRows/offset are the M4 pipeline caps; pass limitRows=0 for
-// "no cap" (drain everything). This is the vec oracle used by every parity case.
+// batch egress. limitRows is the M4 pipeline cap; pass 0 for "no cap" (drain
+// everything).
func runVecPipeline(ctx context.Context, t *testing.T, s *stream, sqo model.StreamQueryOptions,
- desc bool, offset, limitRows uint32,
+ desc bool, limitRows uint32,
) []*streamv1.Element {
t.Helper()
src, err := s.queryVectorized(ctx, sqo)
@@ -92,7 +179,7 @@ func runVecPipeline(ctx context.Context, t *testing.T, s *stream, sqo model.Stre
mergeCap = sqo.MaxElementSize
}
pipeline, err := vstream.BuildStreamMergePipeline(
- &testVecSource{src: src, schema: schema}, schema, desc, offset, limitRows, vstream.DefaultConfig().BatchSize, mergeCap)
+ &testVecSource{src: src, schema: schema}, schema, desc, 0, limitRows, vstream.DefaultConfig().BatchSize, mergeCap)
require.NoError(t, err)
require.NoError(t, pipeline.Init(ctx))
var batches []*vectorized.RecordBatch
@@ -110,39 +197,6 @@ func runVecPipeline(ctx context.Context, t *testing.T, s *stream, sqo model.Stre
return elems
}
-// runRowPath materializes the row oracle's elements: s.Query drains one Pull into
-// a StreamResult, then BuildElementsFromStreamResult converts to elements. The
-// caller controls MaxElementSize via sqo.
-func runRowPath(ctx context.Context, t *testing.T, s *stream, sqo model.StreamQueryOptions) []*streamv1.Element {
- t.Helper()
- rowRes, err := s.Query(ctx, sqo)
- require.NoError(t, err)
- require.NotNil(t, rowRes)
- defer rowRes.Release()
- rowElements, err := logicalstream.BuildElementsFromStreamResult(ctx, rowRes, sqo.TagProjection)
- require.NoError(t, err)
- return rowElements
-}
-
-// assertSetParity asserts the two element slices are the SAME set keyed by
-// elementID with identical values (proto.Equal on tags + timestamp), plus the vec
-// output is monotonically ordered by timestamp per asc/desc. This is the tie-break
-// safe discipline the existing parity tests use: equal-timestamp rows have no
-// cross-path-stable order, so only the set (not the per-index order) is compared.
-func assertSetParity(t *testing.T, want, got []*streamv1.Element, desc bool) {
- t.Helper()
- require.Equal(t, len(want), len(got), "element-set size mismatch")
- wantByID := indexByElementID(t, want)
- gotByID := indexByElementID(t, got)
- require.Equal(t, len(wantByID), len(gotByID), "unique element-id set size mismatch")
- for id, wantElem := range wantByID {
- gotElem, ok := gotByID[id]
- require.True(t, ok, "vec path missing elementID present in row path: %s", id)
- assertElementsEqual(t, wantElem, gotElem)
- }
- assertMonotonicTS(t, got, desc)
-}
-
// assertMonotonicTS asserts the elements are monotonically ordered by timestamp
// (non-decreasing for asc, non-increasing for desc).
func assertMonotonicTS(t *testing.T, elems []*streamv1.Element, desc bool) {
@@ -158,255 +212,14 @@ func assertMonotonicTS(t *testing.T, elems []*streamv1.Element, desc bool) {
}
}
-// assertExactParity asserts the two element slices are identical in ORDER and
-// value. Only valid when the fixture has globally unique timestamps (seriesCount=1)
-// so both paths produce the same deterministic total order with no ties.
-func assertExactParity(t *testing.T, want, got []*streamv1.Element) {
- t.Helper()
- require.Equal(t, len(want), len(got), "element count mismatch")
- for i := range want {
- require.Equal(t, want[i].ElementId, got[i].ElementId, "elementID order mismatch at %d", i)
- assertElementsEqual(t, want[i], got[i])
- }
-}
-
-// TestQueryVectorized_Parity_Order runs the row path (oracle) and the full M4 vec
-// pipeline over the SAME fixture for the vec-eligible no-order shapes: default ts
-// order asc and desc. Comparison is set-based + ts-monotonic (tie-break safe)
-// because the multi-series fixture stamps duplicate timestamps.
-func TestQueryVectorized_Parity_Order(t *testing.T) {
- p := parameter{
- batchCount: 1,
- timestampCount: 20,
- seriesCount: 5,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-parity-order",
- }
- s, tr := buildVecTestStream(t, p)
- entities := parityEntities(p.seriesCount)
- projection := fullProjection()
- ctx := context.Background()
-
- cases := []struct {
- order *index.OrderBy
- name string
- }{
- {name: "no-order-asc", order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC}},
- {name: "no-order-desc", order: &index.OrderBy{Sort: modelv1.Sort_SORT_DESC}},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- desc := tc.order.Sort == modelv1.Sort_SORT_DESC
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: projection,
- Order: tc.order,
- MaxElementSize: math.MaxInt32,
- }
-
- rowElements := runRowPath(ctx, t, s, sqo)
- require.NotEmpty(t, rowElements, "row path produced no elements; fixture is degenerate")
-
- vecElements := runVecPipeline(ctx, t, s, sqo, desc, 0, uint32(len(rowElements)*4))
- assertSetParity(t, rowElements, vecElements, desc)
- })
- }
-}
-
-// TestQueryVectorized_Parity_IndexOrder runs the row index-sort oracle and the vec
-// OrderKey path over a DEDICATED index-order fixture (buildIndexOrderStream) for
-// order-by-indexed-tag asc + desc. The shared generateData fixture cannot serve as
-// an index-sort oracle (its sortable docs use DocID=timestamp, not the element id
-// the row index-sort resolves against, and set no timestampField for the sort
-// query's date range) — see the writer's doc comment. Comparison is set-based (the
-// order key is the tag term; ties on it are path-specific) with tag-value parity.
-func TestQueryVectorized_Parity_IndexOrder(t *testing.T) {
- // The index rule the dedicated fixture stamps its sortable docs with. Its single
- // tag "filter-tag" is the order key resolveOrderTag maps to the projection cell.
- indexRule := &databasev1.IndexRule{
- Metadata: &commonv1.Metadata{Name: "filter-idx", Id: indexOrderRuleID},
- Tags: []string{"filter-tag"},
- }
- s, tr := buildIndexOrderStream(t)
- entities := parityEntities(indexOrderSeriesCount)
- projection := fullProjection()
- ctx := context.Background()
-
- cases := []struct {
- name string
- sort modelv1.Sort
- }{
- {name: "index-order-asc", sort: modelv1.Sort_SORT_ASC},
- {name: "index-order-desc", sort: modelv1.Sort_SORT_DESC},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- desc := tc.sort == modelv1.Sort_SORT_DESC
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: projection,
- Order: &index.OrderBy{Index: indexRule, Sort: tc.sort},
- MaxElementSize: math.MaxInt32,
- }
- rowElements := runRowPath(ctx, t, s, sqo)
- require.NotEmpty(t, rowElements, "row index-sort produced no elements; fixture is degenerate")
- vecElements := runVecPipeline(ctx, t, s, sqo, desc, 0, uint32(len(rowElements)*4))
-
- // Order key is the tag term, not ts — ties on equal tag values order
- // path-specifically, so assert set parity (values + membership) only.
- require.Equal(t, len(rowElements), len(vecElements), "element-set size mismatch")
- rowByID := indexByElementID(t, rowElements)
- vecByID := indexByElementID(t, vecElements)
- require.Equal(t, len(rowByID), len(vecByID))
- for id, wantElem := range rowByID {
- gotElem, ok := vecByID[id]
- require.True(t, ok, "vec path missing elementID present in row path: %s", id)
- assertElementsEqual(t, wantElem, gotElem)
- }
- })
- }
-}
-
-// TestQueryVectorized_Parity_Projection covers projection subsets over the SAME
-// fixture: full projection, a single-tag projection, and a projection of a tag
-// absent from every element (exercising the NullTagValue fill on both paths).
-func TestQueryVectorized_Parity_Projection(t *testing.T) {
- p := parameter{
- batchCount: 1,
- timestampCount: 20,
- seriesCount: 5,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-parity-projection",
- }
- s, tr := buildVecTestStream(t, p)
- entities := parityEntities(p.seriesCount)
- ctx := context.Background()
-
- cases := []struct {
- name string
- projection []model.TagProjection
- }{
- {name: "full", projection: fullProjection()},
- {name: "single-tag", projection: []model.TagProjection{{Family: "benchmark-family", Names: []string{"filter-tag"}}}},
- {
- name: "absent-tag-nulltagvalue",
- // "missing-tag" is declared in the projection but written by no element,
- // so both egresses must NullTagValue-fill its column.
- projection: []model.TagProjection{{Family: "benchmark-family", Names: []string{"entity-tag", "missing-tag"}}},
- },
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: tc.projection,
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxElementSize: math.MaxInt32,
- }
- rowElements := runRowPath(ctx, t, s, sqo)
- require.NotEmpty(t, rowElements, "row path produced no elements; fixture is degenerate")
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, uint32(len(rowElements)*4))
- assertSetParity(t, rowElements, vecElements, false)
- })
- }
-}
-
-// TestQueryVectorized_Parity_OffsetLimit covers the offset/limit egress split over
-// a single-series fixture (globally UNIQUE timestamps), enabling exact order+value
-// parity (no ties). The row oracle and vec pipeline both cap the scan at
-// limit+offset and slice [offset:offset+limit], exactly as the *limit plan node
-// does. Cases: limit-only, offset>0+limit, limit>result (all rows), limit<result
-// (top-N).
-func TestQueryVectorized_Parity_OffsetLimit(t *testing.T) {
- // seriesCount=1 => one element per timestamp => globally unique, totally
- // ordered timestamps => deterministic total order on BOTH paths => exact parity.
- p := parameter{
- batchCount: 1,
- timestampCount: 30,
- seriesCount: 1,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-parity-offsetlimit",
- }
- s, tr := buildVecTestStream(t, p)
- entities := parityEntities(p.seriesCount)
- projection := fullProjection()
- ctx := context.Background()
-
- const total = 30 // timestampCount*seriesCount, all in one batch/segment.
-
- cases := []struct {
- name string
- offset uint32
- limit uint32
- }{
- {name: "limit-only", offset: 0, limit: 10},
- {name: "offset-plus-limit", offset: 5, limit: 10},
- {name: "limit-larger-than-result", offset: 0, limit: total + 20},
- {name: "limit-smaller-than-result-topN", offset: 0, limit: 3},
- {name: "offset-into-tail", offset: total - 2, limit: 10},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- // maxElementSize = limit+offset, mirroring PushDownMaxSize at
- // stream_analyzer.go:94 (the scan-level cap the *limit node feeds down).
- maxSize := int(tc.limit) + int(tc.offset)
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: projection,
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxElementSize: maxSize,
- }
-
- rowAll := runRowPath(ctx, t, s, sqo)
- // Apply the *limit node's egress slice to the row oracle (query.go/Query
- // does not itself apply offset/limit — that is the plan node's job).
- rowSliced := sliceOffsetLimit(rowAll, tc.offset, tc.limit)
-
- // The vec pipeline applies offset/limit inside NewLimit at egress, so pass
- // the SAME offset/limit; no post-slice needed.
- vecElements := runVecPipeline(ctx, t, s, sqo, false, tc.offset, tc.limit)
-
- // Unique timestamps => exact order+value parity is provable.
- assertExactParity(t, rowSliced, vecElements)
- assertMonotonicTS(t, vecElements, false)
- })
- }
-}
-
-// sliceOffsetLimit reproduces the *limit plan node's egress slice
-// (allEntities[offset:offset+limit], clamped) so the row oracle can be compared to
-// the vec pipeline's built-in Limit operator.
-func sliceOffsetLimit(elems []*streamv1.Element, offset, limit uint32) []*streamv1.Element {
- off := int(offset)
- lim := int(limit)
- if len(elems) <= off {
- return []*streamv1.Element{}
- }
- end := off + lim
- if end > len(elems) {
- end = len(elems)
- }
- return elems[off:end]
-}
-
// TestQueryVectorized_Parity_Boundary covers the degenerate result shapes: empty
// (query window with no data), single-element, and a >=2-block fixture so
-// cross-block ordering is exercised. Each asserts set parity + ts monotonicity.
+// cross-block ordering is exercised, against the parity_boundary golden. These
+// shapes sit below the server layer (maxElementSize caps, physical block splits)
+// and are unreachable by the server-level fixtures in test/cases/stream/data/want.
func TestQueryVectorized_Parity_Boundary(t *testing.T) {
+ golden := loadParityGolden(t, "parity_boundary")
+
t.Run("empty-result", func(t *testing.T) {
p := parameter{
batchCount: 1,
@@ -423,7 +236,7 @@ func TestQueryVectorized_Parity_Boundary(t *testing.T) {
// A time range entirely BEFORE the fixture window (which starts at ~now-1h):
// SelectSegments still returns the recent segment but the scan yields no rows
- // in [tr.Start, tr.Start] shifted 100 years back, so both paths are empty.
+ // in [tr.Start, tr.Start] shifted 100 years back, so the result is empty.
emptyStart := tr.Start.Add(-100 * 365 * 24 * time.Hour)
emptyTR := timestamp.NewInclusiveTimeRange(emptyStart, tr.Start.Add(-time.Nanosecond))
sqo := model.StreamQueryOptions{
@@ -434,13 +247,11 @@ func TestQueryVectorized_Parity_Boundary(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
MaxElementSize: math.MaxInt32,
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.Empty(t, rowElements, "expected empty row result for a no-data window")
// Pass a real (non-zero) limit so the emptiness comes from the no-data window,
// not from Limit(0) emitting nothing by construction (which would make this a
// tautology). A large limit lets any wrongly-returned row through and fail.
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, math.MaxUint32)
- require.Empty(t, vecElements, "expected empty vec result for a no-data window")
+ vecElements := runVecPipeline(ctx, t, s, sqo, false, math.MaxUint32)
+ assertGoldenParity(t, golden["empty-result"], vecElements, emptyTR.Start, true, false)
})
t.Run("single-element", func(t *testing.T) {
@@ -465,21 +276,15 @@ func TestQueryVectorized_Parity_Boundary(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
MaxElementSize: math.MaxInt32,
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.Len(t, rowElements, 1, "expected exactly one row element")
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, uint32(len(rowElements)*4))
- assertExactParity(t, rowElements, vecElements)
+ vecElements := runVecPipeline(ctx, t, s, sqo, false, 4)
+ assertGoldenParity(t, golden["single-element"], vecElements, tr.Start, true, false)
})
t.Run("multi-block-cross-block-order", func(t *testing.T) {
// A block splits on series-id change (part.go:204), so seriesCount=3 forces
// >=3 physical blocks within the single part; the vec SortedMerge must order
- // across those blocks identically to the row path's cross-block merge. The
- // element count (3*50=150) stays well under one scan batch round (32 blocks)
- // so the row oracle's ONE non-empty Pull covers the COMPLETE result — the
- // documented one-Pull limitation of BuildElementsFromStreamResult. Multiple
- // series share timestamps (ties), so parity is set-based + ts-monotonic, the
- // same tie-break-safe discipline the existing parity tests use.
+ // across those blocks. Multiple series share timestamps (ties), so the golden
+ // comparison is set-based + ts-monotonic rather than order-exact.
p := parameter{
batchCount: 1,
timestampCount: 50,
@@ -500,73 +305,27 @@ func TestQueryVectorized_Parity_Boundary(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
MaxElementSize: math.MaxInt32,
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.NotEmpty(t, rowElements)
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, uint32(len(rowElements)*4))
- assertSetParity(t, rowElements, vecElements, false)
+ ascCase := golden["multi-block-cross-block-order-asc"]
+ vecElements := runVecPipeline(ctx, t, s, sqo, false, uint32(ascCase.ElementCount*4))
+ assertGoldenParity(t, ascCase, vecElements, tr.Start, false, false)
// Same fixture, DESC, to exercise the desc cross-block merge direction.
sqoDesc := sqo
sqoDesc.Order = &index.OrderBy{Sort: modelv1.Sort_SORT_DESC}
- rowDesc := runRowPath(ctx, t, s, sqoDesc)
- require.NotEmpty(t, rowDesc)
- vecDesc := runVecPipeline(ctx, t, s, sqoDesc, true, 0, uint32(len(rowDesc)*4))
- assertSetParity(t, rowDesc, vecDesc, true)
+ descCase := golden["multi-block-cross-block-order-desc"]
+ vecDesc := runVecPipeline(ctx, t, s, sqoDesc, true, uint32(descCase.ElementCount*4))
+ assertGoldenParity(t, descCase, vecDesc, tr.Start, false, true)
})
}
-// TestQueryVectorized_Parity_Fallback_RowPathCorrect asserts that a shape M6 does
-// NOT vectorize (VecExecutable declines) still returns correct elements via the
-// row path. A tag-filter-wrapped scan (criteria tags needing row-side re-matching)
-// is the canonical non-vectorizable shape: VecExecutable returns nil for it, so
-// the engine runs Query. Here we assert the row path itself is correct on the
-// fixture (the fallback destination), which is the guarantee that matters.
-func TestQueryVectorized_Parity_Fallback_RowPathCorrect(t *testing.T) {
- p := parameter{
- batchCount: 1,
- timestampCount: 20,
- seriesCount: 5,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-parity-fallback",
- }
- s, tr := buildVecTestStream(t, p)
- entities := parityEntities(p.seriesCount)
- ctx := context.Background()
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: fullProjection(),
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxElementSize: math.MaxInt32,
- }
- rowElements := runRowPath(ctx, t, s, sqo)
- require.NotEmpty(t, rowElements, "fallback row path produced no elements; fixture is degenerate")
- // Every element carries the full projection with non-null values (the fixture
- // writes entity-tag + filter-tag on every element), and ts is asc-ordered.
- for _, e := range rowElements {
- require.Len(t, e.TagFamilies, 1)
- require.Equal(t, "benchmark-family", e.TagFamilies[0].Name)
- require.Len(t, e.TagFamilies[0].Tags, 2)
- require.NotEmpty(t, e.ElementId)
- }
- assertMonotonicTS(t, rowElements, false)
-
- // The decline contract itself: VecExecutable returns nil for a plan that is not
- // the vectorizable *limit→*localIndexScan shape, so the engine runs the row path.
- require.Nil(t, logicalstream.VecExecutable(nil), "VecExecutable must decline a non-limit plan")
-}
-
// TestQueryVectorized_Parity_LargeSingleBlock is the C1 regression: a single
// block with MORE than DefaultBatchSize (1024) rows. Stream blocks are capped by
// uncompressed BYTES (2 MiB), not row count, so a block can hold >1024 rows; the
// scan must drain a cursor across multiple batchSize batches. Before the fix the
// scan emitted only the first batchSize rows of a cursor and silently dropped the
-// rest, so the vec set would be a strict subset of the row set. seriesCount=1 =>
-// one series => one physical block => globally unique, totally ordered timestamps
-// => exact order+value parity is provable.
+// rest, so the result would be a strict subset of the golden. seriesCount=1 =>
+// one series => one physical block => globally unique timestamps => the golden
+// pins the exact order.
func TestQueryVectorized_Parity_LargeSingleBlock(t *testing.T) {
const rows = 1500 // > DefaultBatchSize (1024), single series => single block.
p := parameter{
@@ -589,19 +348,15 @@ func TestQueryVectorized_Parity_LargeSingleBlock(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
MaxElementSize: math.MaxInt32,
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.Len(t, rowElements, rows, "fixture must yield every row in one Pull")
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, uint32(rows*4))
- // No row loss: the vec path must return the SAME 1500 rows in the same order.
- assertExactParity(t, rowElements, vecElements)
+ vecElements := runVecPipeline(ctx, t, s, sqo, false, uint32(rows*4))
+ assertGoldenParity(t, loadParityGolden(t, "parity_large_single_block")["asc"], vecElements, tr.Start, true, false)
}
// TestQueryVectorized_Parity_TopN_Desc is the C2 regression for descending
// top-N: ORDER BY ts DESC with MaxElementSize < total must return the NEWEST N
// rows, not the oldest N. Before the fix the scan truncated the first N rows in
// storage (ascending) order before the merge, so a desc query returned the
-// OLDEST N. seriesCount=1 => unique timestamps => exact parity against the row
-// oracle (which caps AFTER its in-order desc heap merge).
+// OLDEST N. seriesCount=1 => unique timestamps => the golden pins the exact order.
func TestQueryVectorized_Parity_TopN_Desc(t *testing.T) {
const total = 100
const topN = 10
@@ -625,24 +380,20 @@ func TestQueryVectorized_Parity_TopN_Desc(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_DESC},
MaxElementSize: topN, // per-node cap < total => in-order top-N.
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.Len(t, rowElements, topN, "row oracle must cap at MaxElementSize (newest N)")
// The vec merge caps at MaxElementSize (topN) in DESC sort order; no client
// offset/limit slice (limit=topN, offset=0) so egress keeps all N.
- vecElements := runVecPipeline(ctx, t, s, sqo, true, 0, uint32(topN))
- // Newest N kept, in desc order, byte-identical to the row oracle.
- assertExactParity(t, rowElements, vecElements)
- assertMonotonicTS(t, vecElements, true)
+ vecElements := runVecPipeline(ctx, t, s, sqo, true, uint32(topN))
+ assertGoldenParity(t, loadParityGolden(t, "parity_topn_desc")["desc"], vecElements, tr.Start, true, true)
}
// TestQueryVectorized_Parity_TopN_MultiSeries_Asc is the C2 cross-series
// regression: an ASC query over MULTIPLE series with MaxElementSize < total.
// Blocks are seriesID-major, so a pre-merge storage-order cap would keep rows
// from the first series(es) only, not the globally-oldest N across all series.
-// The cap must apply AFTER the cross-series merge (in ts order). Multiple series
-// share timestamps (ties), so parity is the set of kept elementIDs + ts
-// monotonicity, the same tie-break-safe discipline the other multi-series cases
-// use.
+// The cap must apply AFTER the cross-series merge (in ts order). The five series
+// share timestamps, so the boundary timestamp has ties and the kept element-id set
+// at that boundary is not path-stable; the golden's invariant is therefore the
+// KEPT TS WINDOW (its largest offset), not the exact id set.
func TestQueryVectorized_Parity_TopN_MultiSeries_Asc(t *testing.T) {
p := parameter{
batchCount: 1,
@@ -665,16 +416,11 @@ func TestQueryVectorized_Parity_TopN_MultiSeries_Asc(t *testing.T) {
Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
MaxElementSize: topN,
}
- rowElements := runRowPath(ctx, t, s, sqo)
- require.Len(t, rowElements, topN, "row oracle must cap at MaxElementSize across series")
- vecElements := runVecPipeline(ctx, t, s, sqo, false, 0, uint32(topN))
- require.Len(t, vecElements, topN, "vec must keep exactly the in-order top-N across series")
- // The kept ts window must match the row oracle's (globally-oldest N): assert the
- // max kept ts on both paths is identical, so vec did not keep later rows the
- // oracle excluded. Ties at the boundary ts can pick different elementIDs, so the
- // boundary ts (not the exact id set) is the cross-path invariant.
- require.Equal(t, maxTS(rowElements), maxTS(vecElements),
- "vec kept a different ts window than the row oracle (wrong cross-series top-N)")
+ want := loadParityGolden(t, "parity_topn_multiseries_asc")["asc"]
+ vecElements := runVecPipeline(ctx, t, s, sqo, false, uint32(topN))
+ require.Len(t, vecElements, want.ElementCount, "vec must keep exactly the in-order top-N across series")
+ require.Equal(t, maxGoldenTS(want), maxTS(vecElements)-tr.Start.UnixNano(),
+ "vec kept a different ts window than the golden (wrong cross-series top-N)")
assertMonotonicTS(t, vecElements, false)
}
@@ -689,17 +435,22 @@ func maxTS(elems []*streamv1.Element) int64 {
return m
}
+// maxGoldenTS returns the largest recorded timestamp offset in a golden case.
+func maxGoldenTS(c parityCase) int64 {
+ var m int64
+ for _, e := range c.Elements {
+ if e.TS > m {
+ m = e.TS
+ }
+ }
+ return m
+}
+
// buildIndexOrderStream builds a stream over a DEDICATED index-order fixture whose
-// sortable index docs are shaped for BOTH oracles:
-// - the ROW index-sort (query_by_idx.go) resolves elements by the sort doc's
-// DocID == elementID and filters the sort query by the seriesIDField and the
-// timestampField date range, so each doc sets DocID=elementID and Timestamp=ts.
-// - the VEC OrderKey path keys on the "filter-tag" tag value (resolveOrderTag),
-// so the element's filter-tag value equals the sortable field's term.
-//
-// The shared generateData fixture cannot serve as an index-sort oracle: it stamps
-// DocID=timestamp (never matching an element id) and sets no Timestamp on its docs
-// (so the sort query's date range matches nothing). Hence this dedicated writer.
+// sortable index docs carry DocID=elementID and Timestamp=ts, and whose element
+// "filter-tag" value equals the sortable field's term so the vec OrderKey path
+// (resolveOrderTag) can key on it. The shared generateData fixture cannot serve
+// here: it stamps DocID=timestamp and sets no Timestamp on its docs.
//
// Sort values are assigned so index order deliberately DIFFERS from ts order (the
// per-series value decreases as ts increases), proving the test exercises the
@@ -735,12 +486,10 @@ func buildIndexOrderStream(t *testing.T) (*stream, timestamp.TimeRange) {
if ts > maxTS {
maxTS = ts
}
- elementIDStr := strconv.Itoa(k) + "-" + strconv.Itoa(j)
- elementID := convert.HashStr(elementIDStr)
+ elementID := convert.HashStr(strconv.Itoa(k) + "-" + strconv.Itoa(j))
// Sort value decreases as ts increases => index order != ts order. Zero-pad
// so lexicographic term order equals numeric order.
- sortRank := indexOrderTSCount - j
- sortValue := filterTagValuePrefix + fmt.Sprintf("%03d", sortRank)
+ sortValue := filterTagValuePrefix + fmt.Sprintf("%03d", indexOrderTSCount-j)
elems.seriesIDs = append(elems.seriesIDs, sid)
elems.timestamps = append(elems.timestamps, ts)
@@ -748,7 +497,7 @@ func buildIndexOrderStream(t *testing.T) (*stream, timestamp.TimeRange) {
elems.tagFamilies = append(elems.tagFamilies, []tagValues{{
tag: "benchmark-family",
values: []*tagValue{
- {tag: "entity-tag", value: []byte(entityTagValuePrefix + strconv.Itoa(k)), valueType: pbv1.ValueTypeStr},
+ {tag: entityTagName, value: []byte(entityTagValuePrefix + strconv.Itoa(k)), valueType: pbv1.ValueTypeStr},
{tag: "filter-tag", value: []byte(sortValue), valueType: pbv1.ValueTypeStr},
},
}})
@@ -774,11 +523,11 @@ func buildIndexOrderStream(t *testing.T) (*stream, timestamp.TimeRange) {
tst.Index().Write(sortDocs)
seg.DecRef()
- entity := &databasev1.Entity{TagNames: []string{"entity-tag"}}
+ entity := &databasev1.Entity{TagNames: []string{entityTagName}}
tagFamily := &databasev1.TagFamilySpec{
Name: "benchmark-family",
Tags: []*databasev1.TagSpec{
- {Name: "entity-tag", Type: databasev1.TagType_TAG_TYPE_STRING},
+ {Name: entityTagName, Type: databasev1.TagType_TAG_TYPE_STRING},
{Name: "filter-tag", Type: databasev1.TagType_TAG_TYPE_STRING},
},
}
@@ -793,7 +542,6 @@ func buildIndexOrderStream(t *testing.T) (*stream, timestamp.TimeRange) {
pm: protector.Nop{},
vectorized: vstream.DefaultConfig(),
}
- s.vectorized.Enabled = true
s.name, s.group = "benchmark", "test"
var is indexSchema
is.parse(schema)
@@ -802,3 +550,100 @@ func buildIndexOrderStream(t *testing.T) (*stream, timestamp.TimeRange) {
s.tsdb.Store(db)
return s, timestamp.NewInclusiveTimeRange(time.Unix(0, minTS), time.Unix(0, maxTS))
}
+
+// runVecHiddenOrderTag mirrors localIndexScan.ExecuteVectorized for an index-order
+// query whose ordered tag is NOT in the client projection: the scan is asked for
+// scanProjection (client tags + the ordered tag) so the OrderKey column can be
+// populated, and the egress materializes only clientProjection, so the ordered tag
+// never reaches the result.
+func runVecHiddenOrderTag(ctx context.Context, t *testing.T, s *stream, sqo model.StreamQueryOptions,
+ scanProjection, clientProjection []model.TagProjection, desc bool,
+) []*streamv1.Element {
+ t.Helper()
+ scanSQO := sqo
+ scanSQO.TagProjection = scanProjection
+ src, err := s.queryVectorized(ctx, scanSQO)
+ require.NoError(t, err)
+ require.NotNil(t, src)
+ schema := src.Schema()
+ pipeline, err := vstream.BuildStreamMergePipeline(
+ &testVecSource{src: src, schema: schema}, schema, desc, 0, math.MaxUint32, vstream.DefaultConfig().BatchSize, 0)
+ require.NoError(t, err)
+ require.NoError(t, pipeline.Init(ctx))
+ var batches []*vectorized.RecordBatch
+ for {
+ batch, nextErr := pipeline.Next(ctx)
+ require.NoError(t, nextErr)
+ if batch == nil {
+ break
+ }
+ batches = append(batches, batch)
+ }
+ elems, err := BuildElementsFromBatches(batches, clientProjection)
+ require.NoError(t, err)
+ require.NoError(t, pipeline.Close())
+ return elems
+}
+
+// TestQueryVectorized_Parity_IndexOrder_TagNotProjected covers the R-1 gap: an
+// index-order query whose ordered tag is absent from the projection. Vec derives
+// its OrderKey from the ordered tag's projected cell, so such a query used to fall
+// back to the row path; it now projects the tag internally and strips it before
+// egress.
+//
+// The expectation is analytic rather than captured: buildIndexOrderStream assigns
+// each element j (1..indexOrderTSCount) the sort term indexOrderTSCount-j and the
+// timestamp base+j seconds, so index-asc order is j descending — the exact REVERSE
+// of timestamp order. A silent timestamp fallback, the failure this closes, would
+// produce the opposite sequence and fail.
+func TestQueryVectorized_Parity_IndexOrder_TagNotProjected(t *testing.T) {
+ indexRule := &databasev1.IndexRule{
+ Metadata: &commonv1.Metadata{Name: "filter-idx", Id: indexOrderRuleID},
+ Tags: []string{"filter-tag"},
+ }
+ s, tr := buildIndexOrderStream(t)
+ ctx := context.Background()
+ // One series, so the order key has no cross-series ties and the total order is
+ // fully determined by the fixture.
+ entities := parityEntities(1)
+ clientProjection := []model.TagProjection{{Family: "benchmark-family", Names: []string{entityTagName}}}
+ scanProjection := []model.TagProjection{{Family: "benchmark-family", Names: []string{entityTagName, "filter-tag"}}}
+
+ for _, tc := range []struct {
+ name string
+ sort modelv1.Sort
+ }{
+ {name: "index-order-asc", sort: modelv1.Sort_SORT_ASC},
+ {name: "index-order-desc", sort: modelv1.Sort_SORT_DESC},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ desc := tc.sort == modelv1.Sort_SORT_DESC
+ sqo := model.StreamQueryOptions{
+ Name: "benchmark",
+ TimeRange: &tr,
+ Entities: entities,
+ TagProjection: clientProjection,
+ Order: &index.OrderBy{Index: indexRule, Sort: tc.sort},
+ MaxElementSize: math.MaxInt32,
+ }
+ vecElements := runVecHiddenOrderTag(ctx, t, s, sqo, scanProjection, clientProjection, desc)
+ require.Len(t, vecElements, indexOrderTSCount, "index-sort produced an unexpected element count")
+
+ for idx, e := range vecElements {
+ // index-asc walks j from indexOrderTSCount down to 1; index-desc walks it up.
+ j := indexOrderTSCount - idx
+ if desc {
+ j = idx + 1
+ }
+ wantID := hex.EncodeToString(convert.Uint64ToBytes(convert.HashStr("1-" + strconv.Itoa(j))))
+ require.Equal(t, wantID, e.ElementId, "element order mismatch at %d", idx)
+ require.Len(t, e.TagFamilies, 1)
+ require.Len(t, e.TagFamilies[0].Tags, 1, "the ordered tag must not reach the egress")
+ require.Equal(t, entityTagName, e.TagFamilies[0].Tags[0].Key)
+ }
+ // The fixture's sort value decreases as ts increases, so index order is
+ // the reverse of ts order — a timestamp fallback would fail here.
+ assertMonotonicTS(t, vecElements, !desc)
+ })
+ }
+}
diff --git a/banyand/stream/query_vectorized_test.go b/banyand/stream/query_vectorized_test.go
index 6963dbc3c..8080ddea2 100644
--- a/banyand/stream/query_vectorized_test.go
+++ b/banyand/stream/query_vectorized_test.go
@@ -26,7 +26,6 @@ import (
"time"
"github.com/stretchr/testify/require"
- "google.golang.org/protobuf/proto"
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
@@ -38,7 +37,6 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/logger"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- logicalstream "github.com/apache/skywalking-banyandb/pkg/query/logical/stream"
"github.com/apache/skywalking-banyandb/pkg/query/model"
"github.com/apache/skywalking-banyandb/pkg/query/vectorized"
vstream "github.com/apache/skywalking-banyandb/pkg/query/vectorized/stream"
@@ -91,7 +89,6 @@ func buildVecTestStream(t *testing.T, p parameter) (*stream, timestamp.TimeRange
pm: protector.Nop{},
vectorized: vstream.DefaultConfig(),
}
- s.vectorized.Enabled = true
s.name, s.group = "benchmark", "test"
var is indexSchema
is.parse(schema)
@@ -157,211 +154,6 @@ func indexByElementID(t *testing.T, elements []*streamv1.Element) map[string]*st
return out
}
-func assertElementsEqual(t *testing.T, want, got *streamv1.Element) {
- require.True(t, proto.Equal(want.Timestamp, got.Timestamp),
- "timestamp mismatch for %s: want %v got %v", want.ElementId, want.Timestamp, got.Timestamp)
- require.Equal(t, len(want.TagFamilies), len(got.TagFamilies), "tag family count mismatch for %s", want.ElementId)
- for i := range want.TagFamilies {
- wf, gf := want.TagFamilies[i], got.TagFamilies[i]
- require.Equal(t, wf.Name, gf.Name)
- require.Equal(t, len(wf.Tags), len(gf.Tags), "tag count mismatch in family %s", wf.Name)
- for j := range wf.Tags {
- require.Equal(t, wf.Tags[j].Key, gf.Tags[j].Key)
- require.True(t, proto.Equal(wf.Tags[j].Value, gf.Tags[j].Value),
- "tag value mismatch for %s.%s.%s", want.ElementId, wf.Name, wf.Tags[j].Key)
- }
- }
-}
-
-// TestQueryVectorized_NoOrder_MatchesRowPath scans a real fixture part with both
-// the row path (Query -> BuildElementsFromStreamResult) and the vec path
-// (queryVectorized -> BuildElementsFromVecBatches) for a no-order query with a
-// two-tag projection, and asserts the produced element SETS are equal (keyed by
-// elementID). Ordering/dedup across sources is M4; comparison is set-based.
-func TestQueryVectorized_NoOrder_MatchesRowPath(t *testing.T) {
- // A single write batch keeps the fixture to one part / one disjoint block
- // group, so the row egress (BuildElementsFromStreamResult reads exactly one
- // non-empty Pull) and the drain-everything vec egress see the identical set.
- // Cross-part multi-Pull merge is a M4 operator concern, not an M3 scan one.
- p := parameter{
- batchCount: 1,
- timestampCount: 20,
- seriesCount: 5,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-parity",
- }
- s, tr := buildVecTestStream(t, p)
- entities := make([][]*modelv1.TagValue, 0, p.seriesCount)
- for i := 1; i <= p.seriesCount; i++ {
- entities = append(entities, []*modelv1.TagValue{
- {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: entityTagValuePrefix + strconv.Itoa(i)}}},
- })
- }
- tagProjection := []model.TagProjection{{
- Family: "benchmark-family",
- Names: []string{"entity-tag", "filter-tag"},
- }}
- // No-order query over the full fixture range (no inverted filter) so the
- // fixture yields a representative, non-degenerate set for the parity check.
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: tagProjection,
- MaxElementSize: math.MaxInt32,
- }
-
- ctx := context.Background()
-
- rowRes, err := s.Query(ctx, sqo)
- require.NoError(t, err)
- require.NotNil(t, rowRes)
- defer rowRes.Release()
- rowElements, err := logicalstream.BuildElementsFromStreamResult(ctx, rowRes, sqo.TagProjection)
- require.NoError(t, err)
-
- src, err := s.queryVectorized(ctx, sqo)
- require.NoError(t, err)
- require.NotNil(t, src)
- defer src.Release()
- vecElements, err := BuildElementsFromVecBatches(ctx, src, sqo.TagProjection)
- require.NoError(t, err)
-
- require.NotEmpty(t, rowElements, "row path produced no elements; fixture is degenerate")
-
- rowByID := indexByElementID(t, rowElements)
- // The vec path does not dedup across cursors in M3; collapse to first-seen
- // per elementID so the SET comparison matches the row path's deduped set.
- vecByID := make(map[string]*streamv1.Element, len(vecElements))
- for _, e := range vecElements {
- if _, ok := vecByID[e.ElementId]; !ok {
- vecByID[e.ElementId] = e
- }
- }
-
- require.Equal(t, len(rowByID), len(vecByID), "element-set size mismatch between row and vec paths")
- for id, wantElem := range rowByID {
- gotElem, ok := vecByID[id]
- require.True(t, ok, "vec path missing elementID present in row path: %s", id)
- assertElementsEqual(t, wantElem, gotElem)
- }
-}
-
-// TestQueryVectorized_FullPipeline_MatchesRowPath is the M6 standalone-parity
-// test: it drives the vec scan through the FULL M4 pipeline (merge → distinct →
-// limit) and the batch egress (BuildElementsFromBatches), exactly as the data-node
-// standalone processor path does, then asserts the produced []*streamv1.Element is
-// identical (order + values) to the row path for representative queries.
-func TestQueryVectorized_FullPipeline_MatchesRowPath(t *testing.T) {
- // A single write batch keeps the fixture to one part so the row egress
- // (BuildElementsFromStreamResult reads exactly one non-empty Pull) sees the
- // same complete set the drain-everything vec pipeline does. Cross-part
- // dedup/merge is covered by the M4 operator unit tests, not this end-to-end
- // engine parity check.
- p := parameter{
- batchCount: 1,
- timestampCount: 20,
- seriesCount: 5,
- tagCardinality: 4,
- startTimestamp: 1,
- endTimestamp: 1000,
- scenario: "vec-full-pipeline",
- }
- s, tr := buildVecTestStream(t, p)
- entities := make([][]*modelv1.TagValue, 0, p.seriesCount)
- for i := 1; i <= p.seriesCount; i++ {
- entities = append(entities, []*modelv1.TagValue{
- {Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: entityTagValuePrefix + strconv.Itoa(i)}}},
- })
- }
- tagProjection := []model.TagProjection{{
- Family: "benchmark-family",
- Names: []string{"entity-tag", "filter-tag"},
- }}
-
- cases := []struct {
- name string
- desc bool
- }{
- {name: "ts-order-asc", desc: false},
- {name: "ts-order-desc", desc: true},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- order := &index.OrderBy{Sort: modelv1.Sort_SORT_ASC}
- if tc.desc {
- order.Sort = modelv1.Sort_SORT_DESC
- }
- sqo := model.StreamQueryOptions{
- Name: "benchmark",
- TimeRange: &tr,
- Entities: entities,
- TagProjection: tagProjection,
- Order: order,
- MaxElementSize: math.MaxInt32,
- }
-
- ctx := context.Background()
-
- rowRes, err := s.Query(ctx, sqo)
- require.NoError(t, err)
- require.NotNil(t, rowRes)
- defer rowRes.Release()
- rowElements, err := logicalstream.BuildElementsFromStreamResult(ctx, rowRes, sqo.TagProjection)
- require.NoError(t, err)
- require.NotEmpty(t, rowElements, "row path produced no elements; fixture is degenerate")
-
- src, err := s.queryVectorized(ctx, sqo)
- require.NoError(t, err)
- require.NotNil(t, src)
- schema := src.Schema()
- pipeline, err := vstream.BuildStreamMergePipeline(
- &testVecSource{src: src, schema: schema}, schema, tc.desc, 0, uint32(len(rowElements)*4), vstream.DefaultConfig().BatchSize, 0)
- require.NoError(t, err)
- require.NoError(t, pipeline.Init(ctx))
- var batches []*vectorized.RecordBatch
- for {
- batch, nextErr := pipeline.Next(ctx)
- require.NoError(t, nextErr)
- if batch == nil {
- break
- }
- batches = append(batches, batch)
- }
- vecElements, err := BuildElementsFromBatches(batches, sqo.TagProjection)
- require.NoError(t, err)
- require.NoError(t, pipeline.Close())
-
- // Equal-timestamp rows have no cross-path-stable tie-break (the row
- // ts-order and the vec SortedMerge tie differently), so parity is
- // asserted as: (1) identical element SET keyed by elementID with
- // identical values, and (2) the vec output is monotonically ordered by
- // timestamp per asc/desc. This is the same set-based discipline the
- // no-order parity test uses.
- require.Equal(t, len(rowElements), len(vecElements), "element count mismatch")
- rowByID := indexByElementID(t, rowElements)
- vecByID := indexByElementID(t, vecElements)
- require.Equal(t, len(rowByID), len(vecByID), "element-set size mismatch")
- for id, wantElem := range rowByID {
- gotElem, ok := vecByID[id]
- require.True(t, ok, "vec path missing elementID present in row path: %s", id)
- assertElementsEqual(t, wantElem, gotElem)
- }
- for i := 1; i < len(vecElements); i++ {
- prev := vecElements[i-1].Timestamp.AsTime().UnixNano()
- cur := vecElements[i].Timestamp.AsTime().UnixNano()
- if tc.desc {
- require.GreaterOrEqual(t, prev, cur, "vec output not desc-ordered by ts at %d", i)
- } else {
- require.LessOrEqual(t, prev, cur, "vec output not asc-ordered by ts at %d", i)
- }
- }
- })
- }
-}
-
// testVecSource adapts a vecScanSource to a vectorized.PullOperator so the test
// can drive the M4 pipeline directly, mirroring the production vecSourceOperator.
type testVecSource struct {
diff --git a/banyand/stream/stream.go b/banyand/stream/stream.go
index e16c86535..6ce5d2d8c 100644
--- a/banyand/stream/stream.go
+++ b/banyand/stream/stream.go
@@ -72,7 +72,6 @@ type Query interface {
type Stream interface {
GetSchema() *databasev1.Stream
GetIndexRules() []*databasev1.IndexRule
- Query(ctx context.Context, opts model.StreamQueryOptions) (model.StreamQueryResult, error)
QueryVectorized(ctx context.Context, opts model.StreamQueryOptions) (executor.StreamVecScanSource, error)
VectorizedConfig() vstream.VectorizedConfig
}
@@ -167,7 +166,7 @@ func (s *stream) QueryVectorized(ctx context.Context, opts model.StreamQueryOpti
func bindVectorizedFlags(flagS *run.FlagSet, cfg *vstream.VectorizedConfig) {
defaults := vstream.DefaultConfig()
flagS.BoolVar(&cfg.Enabled, "stream-vectorized-enabled", defaults.Enabled,
- "enable the vectorized stream query path")
+ "deprecated no-op, removed in 0.13.0; the row query path was removed in 0.12.0 so =false is rejected at startup")
flagS.IntVar(&cfg.BatchSize, "stream-vectorized-batch-size", defaults.BatchSize,
"row count per vectorized stream batch")
flagS.IntVar(&cfg.QueryMemoryMiB, "stream-vectorized-query-memory-mib", defaults.QueryMemoryMiB,
diff --git a/banyand/stream/svc_liaison.go b/banyand/stream/svc_liaison.go
index ce54df3ab..bfab6aed6 100644
--- a/banyand/stream/svc_liaison.go
+++ b/banyand/stream/svc_liaison.go
@@ -162,8 +162,7 @@ func (s *liaison) Role() databasev1.Role {
func (s *liaison) PreRun(ctx context.Context) error {
s.l = logger.GetLogger(s.Name())
- data.SetStreamWireModeRaw(s.option.vectorized.Enabled)
- s.l.Info().Bool("stream_wire_mode_raw", s.option.vectorized.Enabled).Msg("stream wire mode published (liaison)")
+ data.SetStreamWireModeRaw(true)
s.l.Info().Msg("memory protector is initialized in PreRun")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
var err error
diff --git a/banyand/stream/svc_standalone.go b/banyand/stream/svc_standalone.go
index eefc851b1..a901a931b 100644
--- a/banyand/stream/svc_standalone.go
+++ b/banyand/stream/svc_standalone.go
@@ -243,10 +243,9 @@ func (s *standalone) Role() databasev1.Role {
func (s *standalone) PreRun(ctx context.Context) error {
s.l = logger.GetLogger(s.Name())
- // Native columnar wire frame for the data↔liaison query hop follows the
- // vectorized flag (mirrors trace's wire-mode wiring).
- data.SetStreamWireModeRaw(s.option.vectorized.Enabled)
- s.l.Info().Bool("stream_wire_mode_raw", s.option.vectorized.Enabled).Msg("stream wire mode published (standalone)")
+ // The native columnar wire frame is the only stream query hop format since the
+ // row path was removed in 0.12.0.
+ data.SetStreamWireModeRaw(true)
s.l.Info().Msg("memory protector is initialized in PreRun")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
var err error
diff --git a/banyand/stream/testdata/parity_boundary.yaml b/banyand/stream/testdata/parity_boundary.yaml
new file mode 100644
index 000000000..23f9396b3
--- /dev/null
+++ b/banyand/stream/testdata/parity_boundary.yaml
@@ -0,0 +1,330 @@
+# Golden expectation captured from the ROW query path (stream.Query ->
+# BuildElementsFromStreamResult) on branch rowdrop/base, before the row path was
+# removed (apache/skywalking#13998). It cannot be re-captured afterwards.
+#
+# Normalization, forced by the fixture generator:
+# ts - nanosecond offset from the query time range start. generateData anchors
+# element timestamps at time.Now(), so absolute timestamps differ per run.
+# tags - only entity-tag is recorded. The sibling filter-tag is drawn from
+# crypto/rand per run (benchmark_test.go generateRandomNumber), so its
+# value is not reproducible and no golden can pin it.
+# Element order is as the row path returned it.
+name: parity_boundary
+cases:
+ - name: empty-result
+ query: fixture{ts=20,series=5} order=ts-asc max=MaxInt32 time-range=before-fixture
+ element_count: 0
+ elements: []
+ - name: single-element
+ query: fixture{ts=1,series=1} order=ts-asc max=MaxInt32
+ element_count: 1
+ elements:
+ - {id: 228d3fa5008c9ec2, ts: 0, entity-tag: entity1}
+ - name: multi-block-cross-block-order-asc
+ query: fixture{ts=50,series=3} order=ts-asc max=MaxInt32
+ element_count: 150
+ elements:
+ - {id: 228d3fa5008c9ec2, ts: 0, entity-tag: entity1}
+ - {id: a224fe72b38b9e57, ts: 0, entity-tag: entity3}
+ - {id: 537fef459947c66f, ts: 0, entity-tag: entity2}
+ - {id: 1a2698b8a9775f34, ts: 1000000000, entity-tag: entity2}
+ - {id: 5460f49adbe7aba2, ts: 1000000000, entity-tag: entity1}
+ - {id: bb62ce8f2e1f2c09, ts: 1000000000, entity-tag: entity3}
+ - {id: 495e5792411ce777, ts: 2000000000, entity-tag: entity3}
+ - {id: 511172adf1230e52, ts: 2000000000, entity-tag: entity2}
+ - {id: 22e84c6060359bac, ts: 2000000000, entity-tag: entity1}
+ - {id: 9deb1683a072e62a, ts: 3000000000, entity-tag: entity1}
+ - {id: 5e8d522ca93563ed, ts: 3000000000, entity-tag: entity3}
+ - {id: 19a82d369be44096, ts: 3000000000, entity-tag: entity2}
+ - {id: a77d430237b307e6, ts: 4000000000, entity-tag: entity2}
+ - {id: ee7276ee58e4421c, ts: 4000000000, entity-tag: entity1}
+ - {id: 7410f56e4c878bd6, ts: 4000000000, entity-tag: entity3}
+ - {id: 14f994dd7433eaca, ts: 5000000000, entity-tag: entity3}
+ - {id: 7d4be3cf97db3464, ts: 5000000000, entity-tag: entity2}
+ - {id: adc82a95b2af3502, ts: 5000000000, entity-tag: entity1}
+ - {id: 3ea90da193231260, ts: 6000000000, entity-tag: entity1}
+ - {id: f8e8e05cadafd8d9, ts: 6000000000, entity-tag: entity3}
+ - {id: f57a32391d7004a9, ts: 6000000000, entity-tag: entity2}
+ - {id: a9f498de471f0891, ts: 7000000000, entity-tag: entity2}
+ - {id: fd5a4daefd03225a, ts: 7000000000, entity-tag: entity1}
+ - {id: 3dce7d0d340bfdd6, ts: 7000000000, entity-tag: entity3}
+ - {id: 374fb9dfa68a1b53, ts: 8000000000, entity-tag: entity3}
+ - {id: 123d253203c5e4a2, ts: 8000000000, entity-tag: entity2}
+ - {id: fb6b212ba3dc7ea7, ts: 8000000000, entity-tag: entity1}
+ - {id: 6d766b314fd40b34, ts: 9000000000, entity-tag: entity1}
+ - {id: a5e68038b4d82ec3, ts: 9000000000, entity-tag: entity3}
+ - {id: e8d09d21da82d526, ts: 9000000000, entity-tag: entity2}
+ - {id: ba837fbf480472c1, ts: 10000000000, entity-tag: entity2}
+ - {id: 91e4e2156784c272, ts: 10000000000, entity-tag: entity1}
+ - {id: b132201b666f08f7, ts: 10000000000, entity-tag: entity3}
+ - {id: f30febe9aafcd6fe, ts: 11000000000, entity-tag: entity3}
+ - {id: 74bba4d0a1597e98, ts: 11000000000, entity-tag: entity2}
+ - {id: edb5e8dc146350dd, ts: 11000000000, entity-tag: entity1}
+ - {id: 555c506640cfcfc3, ts: 12000000000, entity-tag: entity1}
+ - {id: 0dbcacbca2453af3, ts: 12000000000, entity-tag: entity3}
+ - {id: 849688decd573cd6, ts: 12000000000, entity-tag: entity2}
+ - {id: 545dd1ec512e49da, ts: 13000000000, entity-tag: entity2}
+ - {id: 33507a497f63a71e, ts: 13000000000, entity-tag: entity1}
+ - {id: 0411484482e13014, ts: 13000000000, entity-tag: entity3}
+ - {id: 0944652b232ba1b9, ts: 14000000000, entity-tag: entity3}
+ - {id: 538a49514e286e74, ts: 14000000000, entity-tag: entity2}
+ - {id: 0a7b933003526de6, ts: 14000000000, entity-tag: entity1}
+ - {id: c936542e3375798d, ts: 15000000000, entity-tag: entity1}
+ - {id: b711384af3460f66, ts: 15000000000, entity-tag: entity3}
+ - {id: 4b955e8deb606112, ts: 15000000000, entity-tag: entity2}
+ - {id: f964c8153fd29641, ts: 16000000000, entity-tag: entity2}
+ - {id: 42c3aa0c95b63157, ts: 16000000000, entity-tag: entity1}
+ - {id: faabf50eeb921b98, ts: 16000000000, entity-tag: entity3}
+ - {id: ea1eb1eebfaf43bd, ts: 17000000000, entity-tag: entity3}
+ - {id: 122176f88270e9a7, ts: 17000000000, entity-tag: entity2}
+ - {id: 6ec6e52265e2b749, ts: 17000000000, entity-tag: entity1}
+ - {id: 05d7ebc714ec1904, ts: 18000000000, entity-tag: entity1}
+ - {id: 7fb9ef4213548390, ts: 18000000000, entity-tag: entity3}
+ - {id: 4ea126df756d9ca6, ts: 18000000000, entity-tag: entity2}
+ - {id: 1fa1a8ee540ac9c3, ts: 19000000000, entity-tag: entity2}
+ - {id: 2b2f9043766f183b, ts: 19000000000, entity-tag: entity1}
+ - {id: 0996cb90985447fb, ts: 19000000000, entity-tag: entity3}
+ - {id: 21c3eb0588f9278d, ts: 20000000000, entity-tag: entity3}
+ - {id: 1110170b26607eb2, ts: 20000000000, entity-tag: entity2}
+ - {id: 9107bfaa51165612, ts: 20000000000, entity-tag: entity1}
+ - {id: 1b9babd91585a47c, ts: 21000000000, entity-tag: entity1}
+ - {id: 924bc5e310b156a6, ts: 21000000000, entity-tag: entity3}
+ - {id: f0d86e6fc5e702d4, ts: 21000000000, entity-tag: entity2}
+ - {id: 1311e46d58bdab10, ts: 22000000000, entity-tag: entity2}
+ - {id: 3c697d223fa7e885, ts: 22000000000, entity-tag: entity1}
+ - {id: 1d4d6b907ad475f8, ts: 22000000000, entity-tag: entity3}
+ - {id: 2c993c0850c3317e, ts: 23000000000, entity-tag: entity3}
+ - {id: 45eaa85f9f5fec51, ts: 23000000000, entity-tag: entity2}
+ - {id: 3567f9a8f34203a9, ts: 23000000000, entity-tag: entity1}
+ - {id: 2b8f731453cef970, ts: 24000000000, entity-tag: entity1}
+ - {id: 60e881cddedda910, ts: 24000000000, entity-tag: entity3}
+ - {id: 41680ed673474a54, ts: 24000000000, entity-tag: entity2}
+ - {id: 49e39c658df38645, ts: 25000000000, entity-tag: entity2}
+ - {id: 2f183b0b0f8d5261, ts: 25000000000, entity-tag: entity1}
+ - {id: f60799162b28ed7a, ts: 25000000000, entity-tag: entity3}
+ - {id: db92c085e93ca12f, ts: 26000000000, entity-tag: entity3}
+ - {id: ef6204d53af8ee4d, ts: 26000000000, entity-tag: entity2}
+ - {id: ed5540096f524762, ts: 26000000000, entity-tag: entity1}
+ - {id: a6f8d18913dee3f2, ts: 27000000000, entity-tag: entity1}
+ - {id: 8eb61fd778ab3428, ts: 27000000000, entity-tag: entity3}
+ - {id: 9638ac92db3b58ad, ts: 27000000000, entity-tag: entity2}
+ - {id: 3f337fca4d3c8454, ts: 28000000000, entity-tag: entity2}
+ - {id: 406060d38d735523, ts: 28000000000, entity-tag: entity1}
+ - {id: dc4ffaf3e51baca5, ts: 28000000000, entity-tag: entity3}
+ - {id: 1577256d1b6fb06e, ts: 29000000000, entity-tag: entity3}
+ - {id: e80c313a613e338f, ts: 29000000000, entity-tag: entity2}
+ - {id: 18866d99f755ba44, ts: 29000000000, entity-tag: entity1}
+ - {id: 3bb0718c89b38432, ts: 30000000000, entity-tag: entity1}
+ - {id: e4a47e48d03beed8, ts: 30000000000, entity-tag: entity3}
+ - {id: 5cd08f5a17da7213, ts: 30000000000, entity-tag: entity2}
+ - {id: 970885a858528f35, ts: 31000000000, entity-tag: entity2}
+ - {id: 699a0f8d59202e15, ts: 31000000000, entity-tag: entity1}
+ - {id: 0fee1938c4232bdd, ts: 31000000000, entity-tag: entity3}
+ - {id: 2f9f432c95cb2bc6, ts: 32000000000, entity-tag: entity3}
+ - {id: 53da81bc77a43cbc, ts: 32000000000, entity-tag: entity2}
+ - {id: ce4bc5a30a138f86, ts: 32000000000, entity-tag: entity1}
+ - {id: 96944c8eda8151e0, ts: 33000000000, entity-tag: entity1}
+ - {id: c60efccfd1a42428, ts: 33000000000, entity-tag: entity3}
+ - {id: 2e0b86a10afe759b, ts: 33000000000, entity-tag: entity2}
+ - {id: ffba6b12acb007e6, ts: 34000000000, entity-tag: entity2}
+ - {id: 5668d093d5b7cc2f, ts: 34000000000, entity-tag: entity1}
+ - {id: a8604e0a58887d79, ts: 34000000000, entity-tag: entity3}
+ - {id: 9b84d6c0b1bf5e38, ts: 35000000000, entity-tag: entity3}
+ - {id: d50f9ce47f148bd2, ts: 35000000000, entity-tag: entity2}
+ - {id: 0fcf14df42123dad, ts: 35000000000, entity-tag: entity1}
+ - {id: 04f30549eab83bd5, ts: 36000000000, entity-tag: entity1}
+ - {id: f265647f654bbc56, ts: 36000000000, entity-tag: entity3}
+ - {id: 3bd33ae92dafb301, ts: 36000000000, entity-tag: entity2}
+ - {id: b2d8fe8420dbcbe2, ts: 37000000000, entity-tag: entity2}
+ - {id: 9189b3f0021486f4, ts: 37000000000, entity-tag: entity1}
+ - {id: 42677c5e1e3c178e, ts: 37000000000, entity-tag: entity3}
+ - {id: 21ad6db1d3c41693, ts: 38000000000, entity-tag: entity3}
+ - {id: abe5093ee668c276, ts: 38000000000, entity-tag: entity2}
+ - {id: 97e2351339a07e18, ts: 38000000000, entity-tag: entity1}
+ - {id: f437abd62134714b, ts: 39000000000, entity-tag: entity1}
+ - {id: db8c21ccc01a6008, ts: 39000000000, entity-tag: entity3}
+ - {id: 4422c0b3f63ea9e9, ts: 39000000000, entity-tag: entity2}
+ - {id: 5db7322a9f81d358, ts: 40000000000, entity-tag: entity2}
+ - {id: ea8de10eff5af41e, ts: 40000000000, entity-tag: entity1}
+ - {id: 70b410f3ec853c4f, ts: 40000000000, entity-tag: entity3}
+ - {id: bf460c1e92b56919, ts: 41000000000, entity-tag: entity3}
+ - {id: 834d962903bc58e2, ts: 41000000000, entity-tag: entity2}
+ - {id: ee024b69f0f10022, ts: 41000000000, entity-tag: entity1}
+ - {id: 04e9f43d670bf631, ts: 42000000000, entity-tag: entity1}
+ - {id: 27de4a77cb0b42d6, ts: 42000000000, entity-tag: entity3}
+ - {id: 0f87966b9cb84cd7, ts: 42000000000, entity-tag: entity2}
+ - {id: 0a6332bcb7ac0033, ts: 43000000000, entity-tag: entity2}
+ - {id: dee9609ef19d3085, ts: 43000000000, entity-tag: entity1}
+ - {id: d82e39e3b6ef28a6, ts: 43000000000, entity-tag: entity3}
+ - {id: b356fb88f7d426ad, ts: 44000000000, entity-tag: entity3}
+ - {id: be92964784b1e815, ts: 44000000000, entity-tag: entity2}
+ - {id: deea371dfb0ad026, ts: 44000000000, entity-tag: entity1}
+ - {id: 3860b3ec2cf5bfa3, ts: 45000000000, entity-tag: entity1}
+ - {id: 61c26d859947dab0, ts: 45000000000, entity-tag: entity3}
+ - {id: 22d9b2a1d0fa5a21, ts: 45000000000, entity-tag: entity2}
+ - {id: 7f46e1c438bdc63c, ts: 46000000000, entity-tag: entity2}
+ - {id: 3f8c64b343a946ab, ts: 46000000000, entity-tag: entity1}
+ - {id: e9ab27d66111c009, ts: 46000000000, entity-tag: entity3}
+ - {id: 8031474da4ba3ffc, ts: 47000000000, entity-tag: entity3}
+ - {id: 7e43152b7c3206a3, ts: 47000000000, entity-tag: entity2}
+ - {id: af8914ddef8aeed4, ts: 47000000000, entity-tag: entity1}
+ - {id: a4d7bf05a690c389, ts: 48000000000, entity-tag: entity1}
+ - {id: 837a4cc1ee350744, ts: 48000000000, entity-tag: entity3}
+ - {id: 93b9e2b73d4f04d3, ts: 48000000000, entity-tag: entity2}
+ - {id: a425ab32cb9864d5, ts: 49000000000, entity-tag: entity2}
+ - {id: a625903f89d99fde, ts: 49000000000, entity-tag: entity3}
+ - {id: ffc50fae92f2a495, ts: 49000000000, entity-tag: entity1}
+ - name: multi-block-cross-block-order-desc
+ query: fixture{ts=50,series=3} order=ts-desc max=MaxInt32
+ element_count: 150
+ elements:
+ - {id: ffc50fae92f2a495, ts: 49000000000, entity-tag: entity1}
+ - {id: a625903f89d99fde, ts: 49000000000, entity-tag: entity3}
+ - {id: a425ab32cb9864d5, ts: 49000000000, entity-tag: entity2}
+ - {id: 93b9e2b73d4f04d3, ts: 48000000000, entity-tag: entity2}
+ - {id: a4d7bf05a690c389, ts: 48000000000, entity-tag: entity1}
+ - {id: 837a4cc1ee350744, ts: 48000000000, entity-tag: entity3}
+ - {id: 8031474da4ba3ffc, ts: 47000000000, entity-tag: entity3}
+ - {id: 7e43152b7c3206a3, ts: 47000000000, entity-tag: entity2}
+ - {id: af8914ddef8aeed4, ts: 47000000000, entity-tag: entity1}
+ - {id: 3f8c64b343a946ab, ts: 46000000000, entity-tag: entity1}
+ - {id: e9ab27d66111c009, ts: 46000000000, entity-tag: entity3}
+ - {id: 7f46e1c438bdc63c, ts: 46000000000, entity-tag: entity2}
+ - {id: 22d9b2a1d0fa5a21, ts: 45000000000, entity-tag: entity2}
+ - {id: 3860b3ec2cf5bfa3, ts: 45000000000, entity-tag: entity1}
+ - {id: 61c26d859947dab0, ts: 45000000000, entity-tag: entity3}
+ - {id: b356fb88f7d426ad, ts: 44000000000, entity-tag: entity3}
+ - {id: be92964784b1e815, ts: 44000000000, entity-tag: entity2}
+ - {id: deea371dfb0ad026, ts: 44000000000, entity-tag: entity1}
+ - {id: dee9609ef19d3085, ts: 43000000000, entity-tag: entity1}
+ - {id: d82e39e3b6ef28a6, ts: 43000000000, entity-tag: entity3}
+ - {id: 0a6332bcb7ac0033, ts: 43000000000, entity-tag: entity2}
+ - {id: 0f87966b9cb84cd7, ts: 42000000000, entity-tag: entity2}
+ - {id: 04e9f43d670bf631, ts: 42000000000, entity-tag: entity1}
+ - {id: 27de4a77cb0b42d6, ts: 42000000000, entity-tag: entity3}
+ - {id: bf460c1e92b56919, ts: 41000000000, entity-tag: entity3}
+ - {id: 834d962903bc58e2, ts: 41000000000, entity-tag: entity2}
+ - {id: ee024b69f0f10022, ts: 41000000000, entity-tag: entity1}
+ - {id: ea8de10eff5af41e, ts: 40000000000, entity-tag: entity1}
+ - {id: 70b410f3ec853c4f, ts: 40000000000, entity-tag: entity3}
+ - {id: 5db7322a9f81d358, ts: 40000000000, entity-tag: entity2}
+ - {id: 4422c0b3f63ea9e9, ts: 39000000000, entity-tag: entity2}
+ - {id: f437abd62134714b, ts: 39000000000, entity-tag: entity1}
+ - {id: db8c21ccc01a6008, ts: 39000000000, entity-tag: entity3}
+ - {id: 21ad6db1d3c41693, ts: 38000000000, entity-tag: entity3}
+ - {id: abe5093ee668c276, ts: 38000000000, entity-tag: entity2}
+ - {id: 97e2351339a07e18, ts: 38000000000, entity-tag: entity1}
+ - {id: 9189b3f0021486f4, ts: 37000000000, entity-tag: entity1}
+ - {id: 42677c5e1e3c178e, ts: 37000000000, entity-tag: entity3}
+ - {id: b2d8fe8420dbcbe2, ts: 37000000000, entity-tag: entity2}
+ - {id: 3bd33ae92dafb301, ts: 36000000000, entity-tag: entity2}
+ - {id: 04f30549eab83bd5, ts: 36000000000, entity-tag: entity1}
+ - {id: f265647f654bbc56, ts: 36000000000, entity-tag: entity3}
+ - {id: 9b84d6c0b1bf5e38, ts: 35000000000, entity-tag: entity3}
+ - {id: d50f9ce47f148bd2, ts: 35000000000, entity-tag: entity2}
+ - {id: 0fcf14df42123dad, ts: 35000000000, entity-tag: entity1}
+ - {id: 5668d093d5b7cc2f, ts: 34000000000, entity-tag: entity1}
+ - {id: a8604e0a58887d79, ts: 34000000000, entity-tag: entity3}
+ - {id: ffba6b12acb007e6, ts: 34000000000, entity-tag: entity2}
+ - {id: 2e0b86a10afe759b, ts: 33000000000, entity-tag: entity2}
+ - {id: 96944c8eda8151e0, ts: 33000000000, entity-tag: entity1}
+ - {id: c60efccfd1a42428, ts: 33000000000, entity-tag: entity3}
+ - {id: 2f9f432c95cb2bc6, ts: 32000000000, entity-tag: entity3}
+ - {id: 53da81bc77a43cbc, ts: 32000000000, entity-tag: entity2}
+ - {id: ce4bc5a30a138f86, ts: 32000000000, entity-tag: entity1}
+ - {id: 699a0f8d59202e15, ts: 31000000000, entity-tag: entity1}
+ - {id: 0fee1938c4232bdd, ts: 31000000000, entity-tag: entity3}
+ - {id: 970885a858528f35, ts: 31000000000, entity-tag: entity2}
+ - {id: 5cd08f5a17da7213, ts: 30000000000, entity-tag: entity2}
+ - {id: 3bb0718c89b38432, ts: 30000000000, entity-tag: entity1}
+ - {id: e4a47e48d03beed8, ts: 30000000000, entity-tag: entity3}
+ - {id: 1577256d1b6fb06e, ts: 29000000000, entity-tag: entity3}
+ - {id: e80c313a613e338f, ts: 29000000000, entity-tag: entity2}
+ - {id: 18866d99f755ba44, ts: 29000000000, entity-tag: entity1}
+ - {id: 406060d38d735523, ts: 28000000000, entity-tag: entity1}
+ - {id: dc4ffaf3e51baca5, ts: 28000000000, entity-tag: entity3}
+ - {id: 3f337fca4d3c8454, ts: 28000000000, entity-tag: entity2}
+ - {id: 9638ac92db3b58ad, ts: 27000000000, entity-tag: entity2}
+ - {id: a6f8d18913dee3f2, ts: 27000000000, entity-tag: entity1}
+ - {id: 8eb61fd778ab3428, ts: 27000000000, entity-tag: entity3}
+ - {id: db92c085e93ca12f, ts: 26000000000, entity-tag: entity3}
+ - {id: ef6204d53af8ee4d, ts: 26000000000, entity-tag: entity2}
+ - {id: ed5540096f524762, ts: 26000000000, entity-tag: entity1}
+ - {id: 2f183b0b0f8d5261, ts: 25000000000, entity-tag: entity1}
+ - {id: f60799162b28ed7a, ts: 25000000000, entity-tag: entity3}
+ - {id: 49e39c658df38645, ts: 25000000000, entity-tag: entity2}
+ - {id: 41680ed673474a54, ts: 24000000000, entity-tag: entity2}
+ - {id: 2b8f731453cef970, ts: 24000000000, entity-tag: entity1}
+ - {id: 60e881cddedda910, ts: 24000000000, entity-tag: entity3}
+ - {id: 2c993c0850c3317e, ts: 23000000000, entity-tag: entity3}
+ - {id: 45eaa85f9f5fec51, ts: 23000000000, entity-tag: entity2}
+ - {id: 3567f9a8f34203a9, ts: 23000000000, entity-tag: entity1}
+ - {id: 3c697d223fa7e885, ts: 22000000000, entity-tag: entity1}
+ - {id: 1d4d6b907ad475f8, ts: 22000000000, entity-tag: entity3}
+ - {id: 1311e46d58bdab10, ts: 22000000000, entity-tag: entity2}
+ - {id: f0d86e6fc5e702d4, ts: 21000000000, entity-tag: entity2}
+ - {id: 1b9babd91585a47c, ts: 21000000000, entity-tag: entity1}
+ - {id: 924bc5e310b156a6, ts: 21000000000, entity-tag: entity3}
+ - {id: 21c3eb0588f9278d, ts: 20000000000, entity-tag: entity3}
+ - {id: 1110170b26607eb2, ts: 20000000000, entity-tag: entity2}
+ - {id: 9107bfaa51165612, ts: 20000000000, entity-tag: entity1}
+ - {id: 2b2f9043766f183b, ts: 19000000000, entity-tag: entity1}
+ - {id: 0996cb90985447fb, ts: 19000000000, entity-tag: entity3}
+ - {id: 1fa1a8ee540ac9c3, ts: 19000000000, entity-tag: entity2}
+ - {id: 4ea126df756d9ca6, ts: 18000000000, entity-tag: entity2}
+ - {id: 05d7ebc714ec1904, ts: 18000000000, entity-tag: entity1}
+ - {id: 7fb9ef4213548390, ts: 18000000000, entity-tag: entity3}
+ - {id: ea1eb1eebfaf43bd, ts: 17000000000, entity-tag: entity3}
+ - {id: 122176f88270e9a7, ts: 17000000000, entity-tag: entity2}
+ - {id: 6ec6e52265e2b749, ts: 17000000000, entity-tag: entity1}
+ - {id: 42c3aa0c95b63157, ts: 16000000000, entity-tag: entity1}
+ - {id: faabf50eeb921b98, ts: 16000000000, entity-tag: entity3}
+ - {id: f964c8153fd29641, ts: 16000000000, entity-tag: entity2}
+ - {id: 4b955e8deb606112, ts: 15000000000, entity-tag: entity2}
+ - {id: c936542e3375798d, ts: 15000000000, entity-tag: entity1}
+ - {id: b711384af3460f66, ts: 15000000000, entity-tag: entity3}
+ - {id: 0944652b232ba1b9, ts: 14000000000, entity-tag: entity3}
+ - {id: 538a49514e286e74, ts: 14000000000, entity-tag: entity2}
+ - {id: 0a7b933003526de6, ts: 14000000000, entity-tag: entity1}
+ - {id: 33507a497f63a71e, ts: 13000000000, entity-tag: entity1}
+ - {id: 0411484482e13014, ts: 13000000000, entity-tag: entity3}
+ - {id: 545dd1ec512e49da, ts: 13000000000, entity-tag: entity2}
+ - {id: 849688decd573cd6, ts: 12000000000, entity-tag: entity2}
+ - {id: 555c506640cfcfc3, ts: 12000000000, entity-tag: entity1}
+ - {id: 0dbcacbca2453af3, ts: 12000000000, entity-tag: entity3}
+ - {id: f30febe9aafcd6fe, ts: 11000000000, entity-tag: entity3}
+ - {id: 74bba4d0a1597e98, ts: 11000000000, entity-tag: entity2}
+ - {id: edb5e8dc146350dd, ts: 11000000000, entity-tag: entity1}
+ - {id: 91e4e2156784c272, ts: 10000000000, entity-tag: entity1}
+ - {id: b132201b666f08f7, ts: 10000000000, entity-tag: entity3}
+ - {id: ba837fbf480472c1, ts: 10000000000, entity-tag: entity2}
+ - {id: e8d09d21da82d526, ts: 9000000000, entity-tag: entity2}
+ - {id: 6d766b314fd40b34, ts: 9000000000, entity-tag: entity1}
+ - {id: a5e68038b4d82ec3, ts: 9000000000, entity-tag: entity3}
+ - {id: 374fb9dfa68a1b53, ts: 8000000000, entity-tag: entity3}
+ - {id: 123d253203c5e4a2, ts: 8000000000, entity-tag: entity2}
+ - {id: fb6b212ba3dc7ea7, ts: 8000000000, entity-tag: entity1}
+ - {id: fd5a4daefd03225a, ts: 7000000000, entity-tag: entity1}
+ - {id: 3dce7d0d340bfdd6, ts: 7000000000, entity-tag: entity3}
+ - {id: a9f498de471f0891, ts: 7000000000, entity-tag: entity2}
+ - {id: f57a32391d7004a9, ts: 6000000000, entity-tag: entity2}
+ - {id: 3ea90da193231260, ts: 6000000000, entity-tag: entity1}
+ - {id: f8e8e05cadafd8d9, ts: 6000000000, entity-tag: entity3}
+ - {id: 14f994dd7433eaca, ts: 5000000000, entity-tag: entity3}
+ - {id: 7d4be3cf97db3464, ts: 5000000000, entity-tag: entity2}
+ - {id: adc82a95b2af3502, ts: 5000000000, entity-tag: entity1}
+ - {id: ee7276ee58e4421c, ts: 4000000000, entity-tag: entity1}
+ - {id: 7410f56e4c878bd6, ts: 4000000000, entity-tag: entity3}
+ - {id: a77d430237b307e6, ts: 4000000000, entity-tag: entity2}
+ - {id: 19a82d369be44096, ts: 3000000000, entity-tag: entity2}
+ - {id: 9deb1683a072e62a, ts: 3000000000, entity-tag: entity1}
+ - {id: 5e8d522ca93563ed, ts: 3000000000, entity-tag: entity3}
+ - {id: 495e5792411ce777, ts: 2000000000, entity-tag: entity3}
+ - {id: 511172adf1230e52, ts: 2000000000, entity-tag: entity2}
+ - {id: 22e84c6060359bac, ts: 2000000000, entity-tag: entity1}
+ - {id: 5460f49adbe7aba2, ts: 1000000000, entity-tag: entity1}
+ - {id: bb62ce8f2e1f2c09, ts: 1000000000, entity-tag: entity3}
+ - {id: 1a2698b8a9775f34, ts: 1000000000, entity-tag: entity2}
+ - {id: 537fef459947c66f, ts: 0, entity-tag: entity2}
+ - {id: a224fe72b38b9e57, ts: 0, entity-tag: entity3}
+ - {id: 228d3fa5008c9ec2, ts: 0, entity-tag: entity1}
diff --git a/banyand/stream/testdata/parity_large_single_block.yaml b/banyand/stream/testdata/parity_large_single_block.yaml
new file mode 100644
index 000000000..c80cfd179
--- /dev/null
+++ b/banyand/stream/testdata/parity_large_single_block.yaml
@@ -0,0 +1,1517 @@
+# Golden expectation captured from the ROW query path (stream.Query ->
+# BuildElementsFromStreamResult) on branch rowdrop/base, before the row path was
+# removed (apache/skywalking#13998). It cannot be re-captured afterwards.
+#
+# Normalization, forced by the fixture generator:
+# ts - nanosecond offset from the query time range start. generateData anchors
+# element timestamps at time.Now(), so absolute timestamps differ per run.
+# tags - only entity-tag is recorded. The sibling filter-tag is drawn from
+# crypto/rand per run (benchmark_test.go generateRandomNumber), so its
+# value is not reproducible and no golden can pin it.
+# Element order is as the row path returned it.
+name: parity_large_single_block
+cases:
+ - name: asc
+ query: fixture{ts=1500,series=1} order=ts-asc max=MaxInt32
+ element_count: 1500
+ elements:
+ - {id: 228d3fa5008c9ec2, ts: 0, entity-tag: entity1}
+ - {id: 5460f49adbe7aba2, ts: 1000000000, entity-tag: entity1}
+ - {id: 22e84c6060359bac, ts: 2000000000, entity-tag: entity1}
+ - {id: 9deb1683a072e62a, ts: 3000000000, entity-tag: entity1}
+ - {id: ee7276ee58e4421c, ts: 4000000000, entity-tag: entity1}
+ - {id: adc82a95b2af3502, ts: 5000000000, entity-tag: entity1}
+ - {id: 3ea90da193231260, ts: 6000000000, entity-tag: entity1}
+ - {id: fd5a4daefd03225a, ts: 7000000000, entity-tag: entity1}
+ - {id: fb6b212ba3dc7ea7, ts: 8000000000, entity-tag: entity1}
+ - {id: 6d766b314fd40b34, ts: 9000000000, entity-tag: entity1}
+ - {id: 91e4e2156784c272, ts: 10000000000, entity-tag: entity1}
+ - {id: edb5e8dc146350dd, ts: 11000000000, entity-tag: entity1}
+ - {id: 555c506640cfcfc3, ts: 12000000000, entity-tag: entity1}
+ - {id: 33507a497f63a71e, ts: 13000000000, entity-tag: entity1}
+ - {id: 0a7b933003526de6, ts: 14000000000, entity-tag: entity1}
+ - {id: c936542e3375798d, ts: 15000000000, entity-tag: entity1}
+ - {id: 42c3aa0c95b63157, ts: 16000000000, entity-tag: entity1}
+ - {id: 6ec6e52265e2b749, ts: 17000000000, entity-tag: entity1}
+ - {id: 05d7ebc714ec1904, ts: 18000000000, entity-tag: entity1}
+ - {id: 2b2f9043766f183b, ts: 19000000000, entity-tag: entity1}
+ - {id: 9107bfaa51165612, ts: 20000000000, entity-tag: entity1}
+ - {id: 1b9babd91585a47c, ts: 21000000000, entity-tag: entity1}
+ - {id: 3c697d223fa7e885, ts: 22000000000, entity-tag: entity1}
+ - {id: 3567f9a8f34203a9, ts: 23000000000, entity-tag: entity1}
+ - {id: 2b8f731453cef970, ts: 24000000000, entity-tag: entity1}
+ - {id: 2f183b0b0f8d5261, ts: 25000000000, entity-tag: entity1}
+ - {id: ed5540096f524762, ts: 26000000000, entity-tag: entity1}
+ - {id: a6f8d18913dee3f2, ts: 27000000000, entity-tag: entity1}
+ - {id: 406060d38d735523, ts: 28000000000, entity-tag: entity1}
+ - {id: 18866d99f755ba44, ts: 29000000000, entity-tag: entity1}
+ - {id: 3bb0718c89b38432, ts: 30000000000, entity-tag: entity1}
+ - {id: 699a0f8d59202e15, ts: 31000000000, entity-tag: entity1}
+ - {id: ce4bc5a30a138f86, ts: 32000000000, entity-tag: entity1}
+ - {id: 96944c8eda8151e0, ts: 33000000000, entity-tag: entity1}
+ - {id: 5668d093d5b7cc2f, ts: 34000000000, entity-tag: entity1}
+ - {id: 0fcf14df42123dad, ts: 35000000000, entity-tag: entity1}
+ - {id: 04f30549eab83bd5, ts: 36000000000, entity-tag: entity1}
+ - {id: 9189b3f0021486f4, ts: 37000000000, entity-tag: entity1}
+ - {id: 97e2351339a07e18, ts: 38000000000, entity-tag: entity1}
+ - {id: f437abd62134714b, ts: 39000000000, entity-tag: entity1}
+ - {id: ea8de10eff5af41e, ts: 40000000000, entity-tag: entity1}
+ - {id: ee024b69f0f10022, ts: 41000000000, entity-tag: entity1}
+ - {id: 04e9f43d670bf631, ts: 42000000000, entity-tag: entity1}
+ - {id: dee9609ef19d3085, ts: 43000000000, entity-tag: entity1}
+ - {id: deea371dfb0ad026, ts: 44000000000, entity-tag: entity1}
+ - {id: 3860b3ec2cf5bfa3, ts: 45000000000, entity-tag: entity1}
+ - {id: 3f8c64b343a946ab, ts: 46000000000, entity-tag: entity1}
+ - {id: af8914ddef8aeed4, ts: 47000000000, entity-tag: entity1}
+ - {id: a4d7bf05a690c389, ts: 48000000000, entity-tag: entity1}
+ - {id: ffc50fae92f2a495, ts: 49000000000, entity-tag: entity1}
+ - {id: 4ca23cecdcff147a, ts: 50000000000, entity-tag: entity1}
+ - {id: b9808cff29db496f, ts: 51000000000, entity-tag: entity1}
+ - {id: e34c357cc812553b, ts: 52000000000, entity-tag: entity1}
+ - {id: 9c14f4486e00ea48, ts: 53000000000, entity-tag: entity1}
+ - {id: cbbf06da540a2134, ts: 54000000000, entity-tag: entity1}
+ - {id: 1477db75344704d2, ts: 55000000000, entity-tag: entity1}
+ - {id: 5846a1a0fedc4e50, ts: 56000000000, entity-tag: entity1}
+ - {id: 17428c8839998e4a, ts: 57000000000, entity-tag: entity1}
+ - {id: f969ac8c10567507, ts: 58000000000, entity-tag: entity1}
+ - {id: e65c6a010d250a67, ts: 59000000000, entity-tag: entity1}
+ - {id: b5f269cacb2f4936, ts: 60000000000, entity-tag: entity1}
+ - {id: d6bc311d70a3b553, ts: 61000000000, entity-tag: entity1}
+ - {id: a5425088f79c5225, ts: 62000000000, entity-tag: entity1}
+ - {id: baf1f925844da734, ts: 63000000000, entity-tag: entity1}
+ - {id: 33b3942eb4e99a52, ts: 64000000000, entity-tag: entity1}
+ - {id: 780eb9bf3b359279, ts: 65000000000, entity-tag: entity1}
+ - {id: 9ee6c2fabb5877b7, ts: 66000000000, entity-tag: entity1}
+ - {id: ac0a97a66ffd63c5, ts: 67000000000, entity-tag: entity1}
+ - {id: ca6c45e62482462d, ts: 68000000000, entity-tag: entity1}
+ - {id: bcab7a961eb51812, ts: 69000000000, entity-tag: entity1}
+ - {id: 503df7747309c141, ts: 70000000000, entity-tag: entity1}
+ - {id: 65addb9fc339f7cd, ts: 71000000000, entity-tag: entity1}
+ - {id: b96f2b83fae193eb, ts: 72000000000, entity-tag: entity1}
+ - {id: 25011d6f4ead8c32, ts: 73000000000, entity-tag: entity1}
+ - {id: 10f4317c07911373, ts: 74000000000, entity-tag: entity1}
+ - {id: b5c7bb3cd63fcd9f, ts: 75000000000, entity-tag: entity1}
+ - {id: a0dea2fbd02c672b, ts: 76000000000, entity-tag: entity1}
+ - {id: d1389147dd60a49c, ts: 77000000000, entity-tag: entity1}
+ - {id: 3026946e8827e768, ts: 78000000000, entity-tag: entity1}
+ - {id: a0d9d9f2c7b2ad73, ts: 79000000000, entity-tag: entity1}
+ - {id: 0cab7b8bad2d5cb3, ts: 80000000000, entity-tag: entity1}
+ - {id: c9d1d52f5eb1cc13, ts: 81000000000, entity-tag: entity1}
+ - {id: 1ca5da3f02aa88af, ts: 82000000000, entity-tag: entity1}
+ - {id: 9b1b03821eea8161, ts: 83000000000, entity-tag: entity1}
+ - {id: de9c6bbfafdc4ef4, ts: 84000000000, entity-tag: entity1}
+ - {id: ae97b61d29d9d424, ts: 85000000000, entity-tag: entity1}
+ - {id: 75b38e14ec3acd24, ts: 86000000000, entity-tag: entity1}
+ - {id: 1836f53e80b0c865, ts: 87000000000, entity-tag: entity1}
+ - {id: f4945b2847643719, ts: 88000000000, entity-tag: entity1}
+ - {id: 0f5b36b1adeb8640, ts: 89000000000, entity-tag: entity1}
+ - {id: e4346756ceb39846, ts: 90000000000, entity-tag: entity1}
+ - {id: 79f4994c063b03fe, ts: 91000000000, entity-tag: entity1}
+ - {id: ab77a7816f6952ba, ts: 92000000000, entity-tag: entity1}
+ - {id: 2897044e46e6b512, ts: 93000000000, entity-tag: entity1}
+ - {id: cfb0945e07a4b99e, ts: 94000000000, entity-tag: entity1}
+ - {id: d2688b2e323a5e6c, ts: 95000000000, entity-tag: entity1}
+ - {id: 25faf71b4e2c9b9b, ts: 96000000000, entity-tag: entity1}
+ - {id: e6c327a35bc00991, ts: 97000000000, entity-tag: entity1}
+ - {id: bae7b044cf1f3238, ts: 98000000000, entity-tag: entity1}
+ - {id: 653cbe3860b2f8b8, ts: 99000000000, entity-tag: entity1}
+ - {id: 2f65dac21393f668, ts: 100000000000, entity-tag: entity1}
+ - {id: 7f0dc034cdba26e9, ts: 101000000000, entity-tag: entity1}
+ - {id: 8e20129c6727b437, ts: 102000000000, entity-tag: entity1}
+ - {id: ea49705a5eb33c1f, ts: 103000000000, entity-tag: entity1}
+ - {id: c9ad4a367de90d70, ts: 104000000000, entity-tag: entity1}
+ - {id: 5376a1057ec322e4, ts: 105000000000, entity-tag: entity1}
+ - {id: 3f95a1fb608a8834, ts: 106000000000, entity-tag: entity1}
+ - {id: 45c32ee22a19dffe, ts: 107000000000, entity-tag: entity1}
+ - {id: 34d7bba4a84fdc37, ts: 108000000000, entity-tag: entity1}
+ - {id: 9d55232bcd53bfde, ts: 109000000000, entity-tag: entity1}
+ - {id: 75ca64da8ab49123, ts: 110000000000, entity-tag: entity1}
+ - {id: 735306deaa6f286c, ts: 111000000000, entity-tag: entity1}
+ - {id: 93bd1f8c9fdb7d14, ts: 112000000000, entity-tag: entity1}
+ - {id: d8a09d13ef6f1872, ts: 113000000000, entity-tag: entity1}
+ - {id: 6f5a43e065ac6f6e, ts: 114000000000, entity-tag: entity1}
+ - {id: 469bb05eb4042e09, ts: 115000000000, entity-tag: entity1}
+ - {id: 8e295c5634847dd3, ts: 116000000000, entity-tag: entity1}
+ - {id: 359bdbd9e8db1521, ts: 117000000000, entity-tag: entity1}
+ - {id: eb592b157893e4b0, ts: 118000000000, entity-tag: entity1}
+ - {id: 8fecb9debe679fdf, ts: 119000000000, entity-tag: entity1}
+ - {id: ff933ba253930b9a, ts: 120000000000, entity-tag: entity1}
+ - {id: 4915b383a4755f26, ts: 121000000000, entity-tag: entity1}
+ - {id: a9a107196345f936, ts: 122000000000, entity-tag: entity1}
+ - {id: a3eb7903a418784d, ts: 123000000000, entity-tag: entity1}
+ - {id: a528f7a5f6bef099, ts: 124000000000, entity-tag: entity1}
+ - {id: 7015e2632d278e9f, ts: 125000000000, entity-tag: entity1}
+ - {id: 5cd7c3009d32e7fd, ts: 126000000000, entity-tag: entity1}
+ - {id: 7fa12ce23eaab963, ts: 127000000000, entity-tag: entity1}
+ - {id: 68cc2afad44ca277, ts: 128000000000, entity-tag: entity1}
+ - {id: c0592efffadf93f0, ts: 129000000000, entity-tag: entity1}
+ - {id: a98df56d3ff21289, ts: 130000000000, entity-tag: entity1}
+ - {id: bf463669d05fd3ac, ts: 131000000000, entity-tag: entity1}
+ - {id: d75eeb27d61bb3d7, ts: 132000000000, entity-tag: entity1}
+ - {id: 33e173535e0d5359, ts: 133000000000, entity-tag: entity1}
+ - {id: 86ecc15599c3b4a8, ts: 134000000000, entity-tag: entity1}
+ - {id: 5df7a9d5c58c1520, ts: 135000000000, entity-tag: entity1}
+ - {id: 3bdf73825ed9ace9, ts: 136000000000, entity-tag: entity1}
+ - {id: c5e240a3e2209cc5, ts: 137000000000, entity-tag: entity1}
+ - {id: 4d12ec73ff082349, ts: 138000000000, entity-tag: entity1}
+ - {id: 0126360ab1ddb787, ts: 139000000000, entity-tag: entity1}
+ - {id: 6b8c67b49944139f, ts: 140000000000, entity-tag: entity1}
+ - {id: 2eebda5ea2e78997, ts: 141000000000, entity-tag: entity1}
+ - {id: 5cddbcaafea4ceb8, ts: 142000000000, entity-tag: entity1}
+ - {id: 6e7b38711c8cf746, ts: 143000000000, entity-tag: entity1}
+ - {id: 2178c362c953705b, ts: 144000000000, entity-tag: entity1}
+ - {id: 25a3317ac6980740, ts: 145000000000, entity-tag: entity1}
+ - {id: 8fbcb53860f96263, ts: 146000000000, entity-tag: entity1}
+ - {id: 80c914d1817dd2e0, ts: 147000000000, entity-tag: entity1}
+ - {id: 854fe7092b1b0add, ts: 148000000000, entity-tag: entity1}
+ - {id: 591ef9ec5abe71a7, ts: 149000000000, entity-tag: entity1}
+ - {id: df691d03c03d2e10, ts: 150000000000, entity-tag: entity1}
+ - {id: 1f5bce1624dfdeb6, ts: 151000000000, entity-tag: entity1}
+ - {id: d4c4fe5285b4be3f, ts: 152000000000, entity-tag: entity1}
+ - {id: 27c618b9e38fcce3, ts: 153000000000, entity-tag: entity1}
+ - {id: ca314b940436edd0, ts: 154000000000, entity-tag: entity1}
+ - {id: d8acbf2a6ee47bb1, ts: 155000000000, entity-tag: entity1}
+ - {id: baa331f734030b5a, ts: 156000000000, entity-tag: entity1}
+ - {id: 0a9c7ebcc512d547, ts: 157000000000, entity-tag: entity1}
+ - {id: 18f992aaf110b9b5, ts: 158000000000, entity-tag: entity1}
+ - {id: bc65ecb8f908f591, ts: 159000000000, entity-tag: entity1}
+ - {id: 998f0c6442bb1f08, ts: 160000000000, entity-tag: entity1}
+ - {id: 92a4b606ce277ad1, ts: 161000000000, entity-tag: entity1}
+ - {id: 523c66cf656f9a99, ts: 162000000000, entity-tag: entity1}
+ - {id: 5df508dc95b0f8f3, ts: 163000000000, entity-tag: entity1}
+ - {id: 128ed68ddd132712, ts: 164000000000, entity-tag: entity1}
+ - {id: d1cf688fd19ba7ed, ts: 165000000000, entity-tag: entity1}
+ - {id: cfad9f8f907f42aa, ts: 166000000000, entity-tag: entity1}
+ - {id: d0b02b614a127d61, ts: 167000000000, entity-tag: entity1}
+ - {id: be115a8929f2f7c3, ts: 168000000000, entity-tag: entity1}
+ - {id: 31237992e783266b, ts: 169000000000, entity-tag: entity1}
+ - {id: 44ab97bb0ffe7be4, ts: 170000000000, entity-tag: entity1}
+ - {id: 08af6a39c7115d13, ts: 171000000000, entity-tag: entity1}
+ - {id: 92876d1686c694e3, ts: 172000000000, entity-tag: entity1}
+ - {id: 6a4af97d42308cac, ts: 173000000000, entity-tag: entity1}
+ - {id: 12065b3fc95d45d3, ts: 174000000000, entity-tag: entity1}
+ - {id: 729c99609ea3b0d4, ts: 175000000000, entity-tag: entity1}
+ - {id: 7485fe8695d35e7f, ts: 176000000000, entity-tag: entity1}
+ - {id: 5b3569c34e06d4b4, ts: 177000000000, entity-tag: entity1}
+ - {id: a157b45a7e47abaf, ts: 178000000000, entity-tag: entity1}
+ - {id: 3d9dd749bf0dd3f3, ts: 179000000000, entity-tag: entity1}
+ - {id: 5b3ed09290a37690, ts: 180000000000, entity-tag: entity1}
+ - {id: 9dc7b59bafb44e8c, ts: 181000000000, entity-tag: entity1}
+ - {id: 10adb3f6feeb25be, ts: 182000000000, entity-tag: entity1}
+ - {id: 864b6310b15de6fb, ts: 183000000000, entity-tag: entity1}
+ - {id: 8bdf8e5a9687faa3, ts: 184000000000, entity-tag: entity1}
+ - {id: cecb0b7f309e34ff, ts: 185000000000, entity-tag: entity1}
+ - {id: a928805b0fbaf600, ts: 186000000000, entity-tag: entity1}
+ - {id: 60b9226d52a38da1, ts: 187000000000, entity-tag: entity1}
+ - {id: 750ae708cba9e1c1, ts: 188000000000, entity-tag: entity1}
+ - {id: 7e0ff8d468a22e60, ts: 189000000000, entity-tag: entity1}
+ - {id: 463d9a98183e54f7, ts: 190000000000, entity-tag: entity1}
+ - {id: 8292bd5ce6801774, ts: 191000000000, entity-tag: entity1}
+ - {id: 8e292e4116a0e25c, ts: 192000000000, entity-tag: entity1}
+ - {id: 38a9c59c0ff80437, ts: 193000000000, entity-tag: entity1}
+ - {id: b0ebbdf71ff3b33a, ts: 194000000000, entity-tag: entity1}
+ - {id: d30a9c9f40bff78a, ts: 195000000000, entity-tag: entity1}
+ - {id: 576558b2248ff391, ts: 196000000000, entity-tag: entity1}
+ - {id: f0674a5e220a7354, ts: 197000000000, entity-tag: entity1}
+ - {id: c232045db3397b2f, ts: 198000000000, entity-tag: entity1}
+ - {id: 93e42bcee741f603, ts: 199000000000, entity-tag: entity1}
+ - {id: eb05cda58ccde2cc, ts: 200000000000, entity-tag: entity1}
+ - {id: dd0dfddcca92d59a, ts: 201000000000, entity-tag: entity1}
+ - {id: 244565ce48e79cb1, ts: 202000000000, entity-tag: entity1}
+ - {id: f9b949cabcc76d00, ts: 203000000000, entity-tag: entity1}
+ - {id: c42191bf97c34e9a, ts: 204000000000, entity-tag: entity1}
+ - {id: 7f9f2860a7b6f176, ts: 205000000000, entity-tag: entity1}
+ - {id: f979b64c0ac4a2ba, ts: 206000000000, entity-tag: entity1}
+ - {id: ed23f975921b164d, ts: 207000000000, entity-tag: entity1}
+ - {id: b1ff581d0112f29b, ts: 208000000000, entity-tag: entity1}
+ - {id: 36397dd0cd189788, ts: 209000000000, entity-tag: entity1}
+ - {id: 7f6452fc3bc398e1, ts: 210000000000, entity-tag: entity1}
+ - {id: 48c72076a94038d1, ts: 211000000000, entity-tag: entity1}
+ - {id: f6269878cb66d4f3, ts: 212000000000, entity-tag: entity1}
+ - {id: 6765a33895d1a999, ts: 213000000000, entity-tag: entity1}
+ - {id: 5fb0b734e9933023, ts: 214000000000, entity-tag: entity1}
+ - {id: e5d9d9c6b0d4d745, ts: 215000000000, entity-tag: entity1}
+ - {id: 2a4d5ff3930dcc8b, ts: 216000000000, entity-tag: entity1}
+ - {id: d92cccb553b33e57, ts: 217000000000, entity-tag: entity1}
+ - {id: d7cb231c64d6f015, ts: 218000000000, entity-tag: entity1}
+ - {id: d9ff170fe9a0c41d, ts: 219000000000, entity-tag: entity1}
+ - {id: ea601d134512cc0f, ts: 220000000000, entity-tag: entity1}
+ - {id: 360fd7813111ec75, ts: 221000000000, entity-tag: entity1}
+ - {id: e9dccf9c8a1cca6e, ts: 222000000000, entity-tag: entity1}
+ - {id: fc1335730bb517fa, ts: 223000000000, entity-tag: entity1}
+ - {id: 2341df919e2f4935, ts: 224000000000, entity-tag: entity1}
+ - {id: 906270b8d720ea69, ts: 225000000000, entity-tag: entity1}
+ - {id: c1320bca0c0716c5, ts: 226000000000, entity-tag: entity1}
+ - {id: 8c5a78674ceb9e4e, ts: 227000000000, entity-tag: entity1}
+ - {id: 7cca88e5a1d3108f, ts: 228000000000, entity-tag: entity1}
+ - {id: 5a1a84800c00039b, ts: 229000000000, entity-tag: entity1}
+ - {id: 9e4e417423a1cc4c, ts: 230000000000, entity-tag: entity1}
+ - {id: e595452dfadf894d, ts: 231000000000, entity-tag: entity1}
+ - {id: abb9f42304657a5e, ts: 232000000000, entity-tag: entity1}
+ - {id: d8316e61d84f6ba4, ts: 233000000000, entity-tag: entity1}
+ - {id: dca887f295cb996a, ts: 234000000000, entity-tag: entity1}
+ - {id: 8a83d475c8c76173, ts: 235000000000, entity-tag: entity1}
+ - {id: a4ec8c3fafbe56b2, ts: 236000000000, entity-tag: entity1}
+ - {id: 41e4a8c71e9ad73f, ts: 237000000000, entity-tag: entity1}
+ - {id: 5baa77b1fb7d219a, ts: 238000000000, entity-tag: entity1}
+ - {id: a405daeb5beaa985, ts: 239000000000, entity-tag: entity1}
+ - {id: 4e854d64a630207f, ts: 240000000000, entity-tag: entity1}
+ - {id: 04837f7cfa6024b1, ts: 241000000000, entity-tag: entity1}
+ - {id: d5b75947bc8d201b, ts: 242000000000, entity-tag: entity1}
+ - {id: 57524addf9ecb81a, ts: 243000000000, entity-tag: entity1}
+ - {id: 0304cb6c1804acc2, ts: 244000000000, entity-tag: entity1}
+ - {id: 3073a1ab0b30b996, ts: 245000000000, entity-tag: entity1}
+ - {id: c128d3b2180f6309, ts: 246000000000, entity-tag: entity1}
+ - {id: 73570daca63b3a85, ts: 247000000000, entity-tag: entity1}
+ - {id: 967c6e551ca83585, ts: 248000000000, entity-tag: entity1}
+ - {id: 39a48f21c5681ea0, ts: 249000000000, entity-tag: entity1}
+ - {id: 7777a1bd6033c067, ts: 250000000000, entity-tag: entity1}
+ - {id: 248e4fb933bf60df, ts: 251000000000, entity-tag: entity1}
+ - {id: f50c2a59c517cc9a, ts: 252000000000, entity-tag: entity1}
+ - {id: e2e55deb027382e6, ts: 253000000000, entity-tag: entity1}
+ - {id: 9032234bb0f88744, ts: 254000000000, entity-tag: entity1}
+ - {id: 26488fb98f1bfcb9, ts: 255000000000, entity-tag: entity1}
+ - {id: c6e0c7d88eeeb1b9, ts: 256000000000, entity-tag: entity1}
+ - {id: 81db775af9e56663, ts: 257000000000, entity-tag: entity1}
+ - {id: f23ab7566f57b9a1, ts: 258000000000, entity-tag: entity1}
+ - {id: 8aee94a552d81808, ts: 259000000000, entity-tag: entity1}
+ - {id: f6c077c566babafd, ts: 260000000000, entity-tag: entity1}
+ - {id: d3f50f932c366741, ts: 261000000000, entity-tag: entity1}
+ - {id: c2d8bfdfd7e0c2b4, ts: 262000000000, entity-tag: entity1}
+ - {id: dcf3cbab0707a700, ts: 263000000000, entity-tag: entity1}
+ - {id: 6fbfb66972ac7a78, ts: 264000000000, entity-tag: entity1}
+ - {id: 52ea3112c3198687, ts: 265000000000, entity-tag: entity1}
+ - {id: b63bdfbb2e3376b1, ts: 266000000000, entity-tag: entity1}
+ - {id: 53eefbfc58b32bbf, ts: 267000000000, entity-tag: entity1}
+ - {id: 89269180063af2df, ts: 268000000000, entity-tag: entity1}
+ - {id: 7a787a4316d0fa37, ts: 269000000000, entity-tag: entity1}
+ - {id: acca403ff47e7a88, ts: 270000000000, entity-tag: entity1}
+ - {id: abfd0d32c6af84b6, ts: 271000000000, entity-tag: entity1}
+ - {id: 04e4959d6f74113b, ts: 272000000000, entity-tag: entity1}
+ - {id: 67e4f2737090ef1d, ts: 273000000000, entity-tag: entity1}
+ - {id: 0ec209785c6c775b, ts: 274000000000, entity-tag: entity1}
+ - {id: 8c60f8ad01e9c6fa, ts: 275000000000, entity-tag: entity1}
+ - {id: 4b39aab656e35ffd, ts: 276000000000, entity-tag: entity1}
+ - {id: 33ca2e4266271ccd, ts: 277000000000, entity-tag: entity1}
+ - {id: 2a3cab4b80fbfb50, ts: 278000000000, entity-tag: entity1}
+ - {id: 8bb49024aff58105, ts: 279000000000, entity-tag: entity1}
+ - {id: 8ef9d41bdb41d83f, ts: 280000000000, entity-tag: entity1}
+ - {id: 01bc21f663bce47e, ts: 281000000000, entity-tag: entity1}
+ - {id: bcd6a7475978c0cd, ts: 282000000000, entity-tag: entity1}
+ - {id: 37787139b8d2371d, ts: 283000000000, entity-tag: entity1}
+ - {id: 5a581dc28ab75b9f, ts: 284000000000, entity-tag: entity1}
+ - {id: ae777c1150133dd7, ts: 285000000000, entity-tag: entity1}
+ - {id: 9ce26028e561b215, ts: 286000000000, entity-tag: entity1}
+ - {id: 65127d7480da5d2e, ts: 287000000000, entity-tag: entity1}
+ - {id: 3d82740c50f11cad, ts: 288000000000, entity-tag: entity1}
+ - {id: f75f6fdb7c158f52, ts: 289000000000, entity-tag: entity1}
+ - {id: 86756679ce82f47e, ts: 290000000000, entity-tag: entity1}
+ - {id: 25727cab2b33ed54, ts: 291000000000, entity-tag: entity1}
+ - {id: cd6771711c6b5105, ts: 292000000000, entity-tag: entity1}
+ - {id: de99a17d2f948ce7, ts: 293000000000, entity-tag: entity1}
+ - {id: 2dc888bad5601031, ts: 294000000000, entity-tag: entity1}
+ - {id: fa0c14c155aff5f1, ts: 295000000000, entity-tag: entity1}
+ - {id: 8000b2f5acdc8cb2, ts: 296000000000, entity-tag: entity1}
+ - {id: c9196ce3e2951fd6, ts: 297000000000, entity-tag: entity1}
+ - {id: da374efaec9ef830, ts: 298000000000, entity-tag: entity1}
+ - {id: 8bb656598033697a, ts: 299000000000, entity-tag: entity1}
+ - {id: 4a1a85b5a932848a, ts: 300000000000, entity-tag: entity1}
+ - {id: 87fd9a00bf73e79c, ts: 301000000000, entity-tag: entity1}
+ - {id: 635bfb8970ea63e9, ts: 302000000000, entity-tag: entity1}
+ - {id: 977e72c1a56c2862, ts: 303000000000, entity-tag: entity1}
+ - {id: ff989b548a16ca99, ts: 304000000000, entity-tag: entity1}
+ - {id: a8a03d1b8766b788, ts: 305000000000, entity-tag: entity1}
+ - {id: 613866f05072e3d6, ts: 306000000000, entity-tag: entity1}
+ - {id: 25ebe643825dcb72, ts: 307000000000, entity-tag: entity1}
+ - {id: 493d1f42daba8d2d, ts: 308000000000, entity-tag: entity1}
+ - {id: 5ee75c3c7551cac3, ts: 309000000000, entity-tag: entity1}
+ - {id: a72a401b50a5c3f6, ts: 310000000000, entity-tag: entity1}
+ - {id: 2e29f82a48557277, ts: 311000000000, entity-tag: entity1}
+ - {id: 0bdd4f2592c928d3, ts: 312000000000, entity-tag: entity1}
+ - {id: 8a9074277f392cb1, ts: 313000000000, entity-tag: entity1}
+ - {id: cb01c13d0a21fa73, ts: 314000000000, entity-tag: entity1}
+ - {id: 79cdb9659c19dd09, ts: 315000000000, entity-tag: entity1}
+ - {id: 8f4d75b5661fb097, ts: 316000000000, entity-tag: entity1}
+ - {id: dd6a223849c2f6dc, ts: 317000000000, entity-tag: entity1}
+ - {id: 778d0a1862ca4fe7, ts: 318000000000, entity-tag: entity1}
+ - {id: 76d82443a94e4b0f, ts: 319000000000, entity-tag: entity1}
+ - {id: d92d9c5455be82fd, ts: 320000000000, entity-tag: entity1}
+ - {id: 79bf4a630a68c2f3, ts: 321000000000, entity-tag: entity1}
+ - {id: c3bc98dab34ae2f9, ts: 322000000000, entity-tag: entity1}
+ - {id: e2aca74ed03d0520, ts: 323000000000, entity-tag: entity1}
+ - {id: 0c8395c590e59cb1, ts: 324000000000, entity-tag: entity1}
+ - {id: fd0199fce3064ea0, ts: 325000000000, entity-tag: entity1}
+ - {id: be8f9df2b716f062, ts: 326000000000, entity-tag: entity1}
+ - {id: edcc5ca0f289593f, ts: 327000000000, entity-tag: entity1}
+ - {id: 609551f933ea8d01, ts: 328000000000, entity-tag: entity1}
+ - {id: a79e1981f0f2dd3c, ts: 329000000000, entity-tag: entity1}
+ - {id: 0c54c7e4f8defb60, ts: 330000000000, entity-tag: entity1}
+ - {id: 544f4b2c8b1faa6b, ts: 331000000000, entity-tag: entity1}
+ - {id: fe21dfb8a821537c, ts: 332000000000, entity-tag: entity1}
+ - {id: 25ad611cf3df18a9, ts: 333000000000, entity-tag: entity1}
+ - {id: a498fe88576ff297, ts: 334000000000, entity-tag: entity1}
+ - {id: b6af7774c54411d6, ts: 335000000000, entity-tag: entity1}
+ - {id: 92fe895b873c3e1b, ts: 336000000000, entity-tag: entity1}
+ - {id: e95657ff1a1aab91, ts: 337000000000, entity-tag: entity1}
+ - {id: e4f21d43560fdba3, ts: 338000000000, entity-tag: entity1}
+ - {id: bd1a1f5dec98945e, ts: 339000000000, entity-tag: entity1}
+ - {id: 47912b47dab90475, ts: 340000000000, entity-tag: entity1}
+ - {id: fbb40218986fe2d5, ts: 341000000000, entity-tag: entity1}
+ - {id: 0e68c9169d940ad1, ts: 342000000000, entity-tag: entity1}
+ - {id: 5e39b602ce642d05, ts: 343000000000, entity-tag: entity1}
+ - {id: 3ffb84f0f659a0d1, ts: 344000000000, entity-tag: entity1}
+ - {id: 2536ddcd4c442796, ts: 345000000000, entity-tag: entity1}
+ - {id: e686de7235842e12, ts: 346000000000, entity-tag: entity1}
+ - {id: 3925f72f25348e8d, ts: 347000000000, entity-tag: entity1}
+ - {id: 81cbb2de925315cb, ts: 348000000000, entity-tag: entity1}
+ - {id: 75441ee2c7e2b519, ts: 349000000000, entity-tag: entity1}
+ - {id: 5ff19b3fd1a51de0, ts: 350000000000, entity-tag: entity1}
+ - {id: 89426bf0d5fd28e2, ts: 351000000000, entity-tag: entity1}
+ - {id: 8d837b7c8191f59c, ts: 352000000000, entity-tag: entity1}
+ - {id: 17d77b070796bfbf, ts: 353000000000, entity-tag: entity1}
+ - {id: 069c70bd2ced5e04, ts: 354000000000, entity-tag: entity1}
+ - {id: 622eab77b17bc4f1, ts: 355000000000, entity-tag: entity1}
+ - {id: 02d215c06b39ae53, ts: 356000000000, entity-tag: entity1}
+ - {id: e8def909e6114a6e, ts: 357000000000, entity-tag: entity1}
+ - {id: 2aef521c3e024734, ts: 358000000000, entity-tag: entity1}
+ - {id: 56ae197790fa5132, ts: 359000000000, entity-tag: entity1}
+ - {id: f92dfe8cd6f7ef9b, ts: 360000000000, entity-tag: entity1}
+ - {id: 615b350fb36578b1, ts: 361000000000, entity-tag: entity1}
+ - {id: e4811134c7c3cdc8, ts: 362000000000, entity-tag: entity1}
+ - {id: e324746be1f22176, ts: 363000000000, entity-tag: entity1}
+ - {id: 0e98f32cf20ab98c, ts: 364000000000, entity-tag: entity1}
+ - {id: 2c82f1bcb1b1ae9e, ts: 365000000000, entity-tag: entity1}
+ - {id: 629683d54b945c8a, ts: 366000000000, entity-tag: entity1}
+ - {id: e424df1c85053bef, ts: 367000000000, entity-tag: entity1}
+ - {id: 16c265e2d93e449d, ts: 368000000000, entity-tag: entity1}
+ - {id: 21a3c2ad8038ffe8, ts: 369000000000, entity-tag: entity1}
+ - {id: 3c18fac658841956, ts: 370000000000, entity-tag: entity1}
+ - {id: 170c5498378dd487, ts: 371000000000, entity-tag: entity1}
+ - {id: f71203d4417e9dac, ts: 372000000000, entity-tag: entity1}
+ - {id: f625dca91afbcd3d, ts: 373000000000, entity-tag: entity1}
+ - {id: 9e1ecc57b823375a, ts: 374000000000, entity-tag: entity1}
+ - {id: 65e12a9ec3004da0, ts: 375000000000, entity-tag: entity1}
+ - {id: 8e2a2b875fb3fd5e, ts: 376000000000, entity-tag: entity1}
+ - {id: 805422c39d4c01fb, ts: 377000000000, entity-tag: entity1}
+ - {id: ddb1445362a7b6c6, ts: 378000000000, entity-tag: entity1}
+ - {id: 88347cfee7fcf8d4, ts: 379000000000, entity-tag: entity1}
+ - {id: 183d11f2fca1b43f, ts: 380000000000, entity-tag: entity1}
+ - {id: 82cfcf12f8ac789c, ts: 381000000000, entity-tag: entity1}
+ - {id: 066ed298c728cfbe, ts: 382000000000, entity-tag: entity1}
+ - {id: 2149ae21a11f2ef1, ts: 383000000000, entity-tag: entity1}
+ - {id: 1bd797aa63db0ad3, ts: 384000000000, entity-tag: entity1}
+ - {id: dae4f29c9a1ec18b, ts: 385000000000, entity-tag: entity1}
+ - {id: 94a6857e931e193c, ts: 386000000000, entity-tag: entity1}
+ - {id: 980d208d28b50d43, ts: 387000000000, entity-tag: entity1}
+ - {id: 31fe3fdd0118370d, ts: 388000000000, entity-tag: entity1}
+ - {id: e3028fb466eb4a94, ts: 389000000000, entity-tag: entity1}
+ - {id: 44e6eb15331d7b6f, ts: 390000000000, entity-tag: entity1}
+ - {id: 5125897ecf05d45b, ts: 391000000000, entity-tag: entity1}
+ - {id: 52564d6b88834880, ts: 392000000000, entity-tag: entity1}
+ - {id: 66c0b29f9f4083d8, ts: 393000000000, entity-tag: entity1}
+ - {id: fac30e0ea1c8d8d5, ts: 394000000000, entity-tag: entity1}
+ - {id: 080c10b4bd8a90bd, ts: 395000000000, entity-tag: entity1}
+ - {id: 7510cf002d1ce509, ts: 396000000000, entity-tag: entity1}
+ - {id: ea391d5bc0aaff76, ts: 397000000000, entity-tag: entity1}
+ - {id: ce7fcf5e155448e7, ts: 398000000000, entity-tag: entity1}
+ - {id: ed903d1bd2e10fa3, ts: 399000000000, entity-tag: entity1}
+ - {id: 47ffe489e8d7cc0f, ts: 400000000000, entity-tag: entity1}
+ - {id: e09363a59d8add1f, ts: 401000000000, entity-tag: entity1}
+ - {id: ca0766063625f681, ts: 402000000000, entity-tag: entity1}
+ - {id: cd796238c92f32da, ts: 403000000000, entity-tag: entity1}
+ - {id: ed2cab2dd39d9408, ts: 404000000000, entity-tag: entity1}
+ - {id: 1f3127d37dcab537, ts: 405000000000, entity-tag: entity1}
+ - {id: 70962e1472ad3071, ts: 406000000000, entity-tag: entity1}
+ - {id: b21c78b63923abde, ts: 407000000000, entity-tag: entity1}
+ - {id: 171323dd43cf44cd, ts: 408000000000, entity-tag: entity1}
+ - {id: ccc91c8c54f0495b, ts: 409000000000, entity-tag: entity1}
+ - {id: fc47e58376ce2799, ts: 410000000000, entity-tag: entity1}
+ - {id: 1d25aed1a0d0e355, ts: 411000000000, entity-tag: entity1}
+ - {id: 64077a0a274284f3, ts: 412000000000, entity-tag: entity1}
+ - {id: 4ddd6e4d25246599, ts: 413000000000, entity-tag: entity1}
+ - {id: b9b6c1540c37cee3, ts: 414000000000, entity-tag: entity1}
+ - {id: 5c14b24f8f09d632, ts: 415000000000, entity-tag: entity1}
+ - {id: 650692c8b909be69, ts: 416000000000, entity-tag: entity1}
+ - {id: f4a5c36b2b7c40ce, ts: 417000000000, entity-tag: entity1}
+ - {id: e56cc0cb1b0f81ca, ts: 418000000000, entity-tag: entity1}
+ - {id: 162f84e3e1899387, ts: 419000000000, entity-tag: entity1}
+ - {id: 4cc1f226907ec29e, ts: 420000000000, entity-tag: entity1}
+ - {id: 1b0706c8015883ba, ts: 421000000000, entity-tag: entity1}
+ - {id: 6e88d00d736784bc, ts: 422000000000, entity-tag: entity1}
+ - {id: 1cbb69e436c6aa07, ts: 423000000000, entity-tag: entity1}
+ - {id: 9c1557d7b41adf77, ts: 424000000000, entity-tag: entity1}
+ - {id: 3068f2e128ff8609, ts: 425000000000, entity-tag: entity1}
+ - {id: 81ae004767e784f6, ts: 426000000000, entity-tag: entity1}
+ - {id: 42af3c588b018432, ts: 427000000000, entity-tag: entity1}
+ - {id: 9e25e5519ffd526e, ts: 428000000000, entity-tag: entity1}
+ - {id: bac0a6d63049ec93, ts: 429000000000, entity-tag: entity1}
+ - {id: ac6bcc681b871cad, ts: 430000000000, entity-tag: entity1}
+ - {id: 87717d0cd4cbcc95, ts: 431000000000, entity-tag: entity1}
+ - {id: 6bb522f45afb550b, ts: 432000000000, entity-tag: entity1}
+ - {id: 03acb321c3782cd5, ts: 433000000000, entity-tag: entity1}
+ - {id: d3739a595dbdb139, ts: 434000000000, entity-tag: entity1}
+ - {id: c2d0d7f2ebc0f35f, ts: 435000000000, entity-tag: entity1}
+ - {id: d1f4ee08435e1ef4, ts: 436000000000, entity-tag: entity1}
+ - {id: 2859b9ea07376b84, ts: 437000000000, entity-tag: entity1}
+ - {id: aedb416b1e0c565b, ts: 438000000000, entity-tag: entity1}
+ - {id: c5c6915c4f0b580a, ts: 439000000000, entity-tag: entity1}
+ - {id: abbb3c5707e7cec3, ts: 440000000000, entity-tag: entity1}
+ - {id: 051e6707c1e140bd, ts: 441000000000, entity-tag: entity1}
+ - {id: 265b69684babb36b, ts: 442000000000, entity-tag: entity1}
+ - {id: baa1eb04edb5f8ff, ts: 443000000000, entity-tag: entity1}
+ - {id: b7608540c8625755, ts: 444000000000, entity-tag: entity1}
+ - {id: 713d06eccb80fd08, ts: 445000000000, entity-tag: entity1}
+ - {id: 8b99ef58a4e2d1b1, ts: 446000000000, entity-tag: entity1}
+ - {id: 316660a8b31efc2a, ts: 447000000000, entity-tag: entity1}
+ - {id: b7aa95b6f797aa48, ts: 448000000000, entity-tag: entity1}
+ - {id: c1db6b0364f0b582, ts: 449000000000, entity-tag: entity1}
+ - {id: 7ae26bb366d101ae, ts: 450000000000, entity-tag: entity1}
+ - {id: db4982a4f7dbb23a, ts: 451000000000, entity-tag: entity1}
+ - {id: 82d55033c5ded71a, ts: 452000000000, entity-tag: entity1}
+ - {id: 906c6f4f1ff0a0dd, ts: 453000000000, entity-tag: entity1}
+ - {id: 617db4d91c02fb12, ts: 454000000000, entity-tag: entity1}
+ - {id: c09590b64f547e5e, ts: 455000000000, entity-tag: entity1}
+ - {id: fdc5d5122e6675c5, ts: 456000000000, entity-tag: entity1}
+ - {id: 232fe00b95b05bd9, ts: 457000000000, entity-tag: entity1}
+ - {id: 2bd8da5800b019e0, ts: 458000000000, entity-tag: entity1}
+ - {id: 16a08fe6b121b907, ts: 459000000000, entity-tag: entity1}
+ - {id: e0b9456c699e3ef9, ts: 460000000000, entity-tag: entity1}
+ - {id: 3a2fa609b9540495, ts: 461000000000, entity-tag: entity1}
+ - {id: 1e2989a5637d1530, ts: 462000000000, entity-tag: entity1}
+ - {id: dd19e89236832830, ts: 463000000000, entity-tag: entity1}
+ - {id: ee255b5f5816ec61, ts: 464000000000, entity-tag: entity1}
+ - {id: 2efdcf3f6f65dc1e, ts: 465000000000, entity-tag: entity1}
+ - {id: 869b77a4848bcb09, ts: 466000000000, entity-tag: entity1}
+ - {id: fb95cdd6c948d650, ts: 467000000000, entity-tag: entity1}
+ - {id: 2c0c084a39fc9bb2, ts: 468000000000, entity-tag: entity1}
+ - {id: b590b1100a919940, ts: 469000000000, entity-tag: entity1}
+ - {id: 69e8a395902918b4, ts: 470000000000, entity-tag: entity1}
+ - {id: 504e9d0664f9bf20, ts: 471000000000, entity-tag: entity1}
+ - {id: 416319c854ff187e, ts: 472000000000, entity-tag: entity1}
+ - {id: 2795ca3187a0c02c, ts: 473000000000, entity-tag: entity1}
+ - {id: cfcd060a06942c56, ts: 474000000000, entity-tag: entity1}
+ - {id: 1abec33dc9cd5049, ts: 475000000000, entity-tag: entity1}
+ - {id: 8c2679aec2728002, ts: 476000000000, entity-tag: entity1}
+ - {id: e9c6bbc60e02b2a7, ts: 477000000000, entity-tag: entity1}
+ - {id: 278acba28f7fa960, ts: 478000000000, entity-tag: entity1}
+ - {id: 4bf86fb5944c81e8, ts: 479000000000, entity-tag: entity1}
+ - {id: d4baaae43076bc29, ts: 480000000000, entity-tag: entity1}
+ - {id: cae0fa9f6e439f55, ts: 481000000000, entity-tag: entity1}
+ - {id: 6ee3adeca9485bcc, ts: 482000000000, entity-tag: entity1}
+ - {id: dec7e2010294ba4a, ts: 483000000000, entity-tag: entity1}
+ - {id: c7c28a34cce347b5, ts: 484000000000, entity-tag: entity1}
+ - {id: 6f62fdf2c4a0086d, ts: 485000000000, entity-tag: entity1}
+ - {id: c7dd62abac470414, ts: 486000000000, entity-tag: entity1}
+ - {id: a929b337fe1c9611, ts: 487000000000, entity-tag: entity1}
+ - {id: 645b047cf464e88b, ts: 488000000000, entity-tag: entity1}
+ - {id: 3b904f8c5082b669, ts: 489000000000, entity-tag: entity1}
+ - {id: 24e92789524e1f7f, ts: 490000000000, entity-tag: entity1}
+ - {id: 74f870e9d8fff8c3, ts: 491000000000, entity-tag: entity1}
+ - {id: fc634e0eb2a68d45, ts: 492000000000, entity-tag: entity1}
+ - {id: e936891c73c026e9, ts: 493000000000, entity-tag: entity1}
+ - {id: 75fd00796ca9eeaa, ts: 494000000000, entity-tag: entity1}
+ - {id: 71be687678d5addc, ts: 495000000000, entity-tag: entity1}
+ - {id: 1dee707624f068e3, ts: 496000000000, entity-tag: entity1}
+ - {id: 37241dd5b9013e48, ts: 497000000000, entity-tag: entity1}
+ - {id: 25a73af176429126, ts: 498000000000, entity-tag: entity1}
+ - {id: 06bbcfa5a8003e64, ts: 499000000000, entity-tag: entity1}
+ - {id: 36b4a33aa7a6108b, ts: 500000000000, entity-tag: entity1}
+ - {id: 47f0a7d8a25394d1, ts: 501000000000, entity-tag: entity1}
+ - {id: 403688844f46ee98, ts: 502000000000, entity-tag: entity1}
+ - {id: 83b152e97e5b5583, ts: 503000000000, entity-tag: entity1}
+ - {id: 03988ac737a924d3, ts: 504000000000, entity-tag: entity1}
+ - {id: 7dd5a918bd4c054f, ts: 505000000000, entity-tag: entity1}
+ - {id: f9ad1cc0defcb6d4, ts: 506000000000, entity-tag: entity1}
+ - {id: c721a71db52535b4, ts: 507000000000, entity-tag: entity1}
+ - {id: 0f4549fabc70d86e, ts: 508000000000, entity-tag: entity1}
+ - {id: 24d71d10d1126dff, ts: 509000000000, entity-tag: entity1}
+ - {id: 985df9f9636ee2a3, ts: 510000000000, entity-tag: entity1}
+ - {id: c01fe80a5d4305b0, ts: 511000000000, entity-tag: entity1}
+ - {id: 0737b48081595e81, ts: 512000000000, entity-tag: entity1}
+ - {id: 9fc6059a0da24c81, ts: 513000000000, entity-tag: entity1}
+ - {id: 66a2630a785862a0, ts: 514000000000, entity-tag: entity1}
+ - {id: 23644ff5baf60545, ts: 515000000000, entity-tag: entity1}
+ - {id: 16ab87d6ecb38adf, ts: 516000000000, entity-tag: entity1}
+ - {id: bfb30d7e6fafbca9, ts: 517000000000, entity-tag: entity1}
+ - {id: 20eb5dc59ef4b82e, ts: 518000000000, entity-tag: entity1}
+ - {id: 0d18d873cab9b115, ts: 519000000000, entity-tag: entity1}
+ - {id: 8d7310bef11209f0, ts: 520000000000, entity-tag: entity1}
+ - {id: 7f18e1f41985413e, ts: 521000000000, entity-tag: entity1}
+ - {id: 33cf06b3e700f500, ts: 522000000000, entity-tag: entity1}
+ - {id: bac3ca1eb2ccdef9, ts: 523000000000, entity-tag: entity1}
+ - {id: f9106b91de854d0e, ts: 524000000000, entity-tag: entity1}
+ - {id: 444bfa25bc4a9c47, ts: 525000000000, entity-tag: entity1}
+ - {id: 66e64c94769418b7, ts: 526000000000, entity-tag: entity1}
+ - {id: 1105bbe1452cbf57, ts: 527000000000, entity-tag: entity1}
+ - {id: c227a61e28dd15de, ts: 528000000000, entity-tag: entity1}
+ - {id: 0e07fad5ae411559, ts: 529000000000, entity-tag: entity1}
+ - {id: 4522a3e705ba91d1, ts: 530000000000, entity-tag: entity1}
+ - {id: 7c0ef02c0f3c3e15, ts: 531000000000, entity-tag: entity1}
+ - {id: 118e256024a2481b, ts: 532000000000, entity-tag: entity1}
+ - {id: fe356c3f3bef766c, ts: 533000000000, entity-tag: entity1}
+ - {id: 248221cb0112bab1, ts: 534000000000, entity-tag: entity1}
+ - {id: 79e68375e31daf05, ts: 535000000000, entity-tag: entity1}
+ - {id: 5298973536c6b685, ts: 536000000000, entity-tag: entity1}
+ - {id: 00970177c3841620, ts: 537000000000, entity-tag: entity1}
+ - {id: aa65a3cd37b9f62a, ts: 538000000000, entity-tag: entity1}
+ - {id: 28d3b3bdd95a9571, ts: 539000000000, entity-tag: entity1}
+ - {id: 2886ac5b9977f46d, ts: 540000000000, entity-tag: entity1}
+ - {id: 25243c2e7571dbf5, ts: 541000000000, entity-tag: entity1}
+ - {id: 0c6fbe5b2170b2d7, ts: 542000000000, entity-tag: entity1}
+ - {id: bd49c5447574e552, ts: 543000000000, entity-tag: entity1}
+ - {id: 87d14593d68adfd0, ts: 544000000000, entity-tag: entity1}
+ - {id: a2b24d334072edef, ts: 545000000000, entity-tag: entity1}
+ - {id: 03fa7a0da77ca2ce, ts: 546000000000, entity-tag: entity1}
+ - {id: aef1ac6e86d008b9, ts: 547000000000, entity-tag: entity1}
+ - {id: 3c4b8af81fc96b4b, ts: 548000000000, entity-tag: entity1}
+ - {id: 29a876e1c96671eb, ts: 549000000000, entity-tag: entity1}
+ - {id: a2a870158dbe91e9, ts: 550000000000, entity-tag: entity1}
+ - {id: bdf6db8a56753631, ts: 551000000000, entity-tag: entity1}
+ - {id: b8d33bf5c656c74a, ts: 552000000000, entity-tag: entity1}
+ - {id: 570381efc29f4371, ts: 553000000000, entity-tag: entity1}
+ - {id: 1fc615365f438b87, ts: 554000000000, entity-tag: entity1}
+ - {id: e05b3a40956872d6, ts: 555000000000, entity-tag: entity1}
+ - {id: 7d8ac562c8f1c28d, ts: 556000000000, entity-tag: entity1}
+ - {id: e521b142c5934f6a, ts: 557000000000, entity-tag: entity1}
+ - {id: 9b5ca28a5fa7120e, ts: 558000000000, entity-tag: entity1}
+ - {id: f96413f60573b425, ts: 559000000000, entity-tag: entity1}
+ - {id: d23cbd972f840909, ts: 560000000000, entity-tag: entity1}
+ - {id: b437164587b502db, ts: 561000000000, entity-tag: entity1}
+ - {id: 87425911b77090a0, ts: 562000000000, entity-tag: entity1}
+ - {id: 0fb9dc923dcd4228, ts: 563000000000, entity-tag: entity1}
+ - {id: bb8a3bc294d6e40e, ts: 564000000000, entity-tag: entity1}
+ - {id: 647b85af72688c1c, ts: 565000000000, entity-tag: entity1}
+ - {id: 422358b9c58c60b5, ts: 566000000000, entity-tag: entity1}
+ - {id: 755d3432b6c58f7d, ts: 567000000000, entity-tag: entity1}
+ - {id: b8959bc79254bf2f, ts: 568000000000, entity-tag: entity1}
+ - {id: f4e4170ecbfd38ef, ts: 569000000000, entity-tag: entity1}
+ - {id: a29f667bf86fbf87, ts: 570000000000, entity-tag: entity1}
+ - {id: 234ea03db5da0e86, ts: 571000000000, entity-tag: entity1}
+ - {id: 0542745ce33ddafa, ts: 572000000000, entity-tag: entity1}
+ - {id: bcc9be59802cea3c, ts: 573000000000, entity-tag: entity1}
+ - {id: 1f465f0998b677ce, ts: 574000000000, entity-tag: entity1}
+ - {id: 37fd643c3ba4436a, ts: 575000000000, entity-tag: entity1}
+ - {id: a4960c0e7e4a9fa6, ts: 576000000000, entity-tag: entity1}
+ - {id: ad3c67fbfe390f72, ts: 577000000000, entity-tag: entity1}
+ - {id: eec781a3c45bc4e1, ts: 578000000000, entity-tag: entity1}
+ - {id: d0a460d0b2cc4821, ts: 579000000000, entity-tag: entity1}
+ - {id: c4e553acd8bbb655, ts: 580000000000, entity-tag: entity1}
+ - {id: 15d5ad9a502a124a, ts: 581000000000, entity-tag: entity1}
+ - {id: ec69d80fe1682714, ts: 582000000000, entity-tag: entity1}
+ - {id: 6eb809dbc1d1f822, ts: 583000000000, entity-tag: entity1}
+ - {id: c6b35753790521b5, ts: 584000000000, entity-tag: entity1}
+ - {id: 9e1593f6eb42a696, ts: 585000000000, entity-tag: entity1}
+ - {id: 0c0e1ce58ac73978, ts: 586000000000, entity-tag: entity1}
+ - {id: f86160970117ff6d, ts: 587000000000, entity-tag: entity1}
+ - {id: ae170b071e900d81, ts: 588000000000, entity-tag: entity1}
+ - {id: d7510dcc97cec406, ts: 589000000000, entity-tag: entity1}
+ - {id: aa03847abadba32e, ts: 590000000000, entity-tag: entity1}
+ - {id: 01aa326b51505826, ts: 591000000000, entity-tag: entity1}
+ - {id: bcc480b1455223ff, ts: 592000000000, entity-tag: entity1}
+ - {id: 8ce044dc27d86910, ts: 593000000000, entity-tag: entity1}
+ - {id: c2982fe23ba47378, ts: 594000000000, entity-tag: entity1}
+ - {id: 8951dbccf097295b, ts: 595000000000, entity-tag: entity1}
+ - {id: f0cd1ba6ef237c42, ts: 596000000000, entity-tag: entity1}
+ - {id: a2d4fc404165732e, ts: 597000000000, entity-tag: entity1}
+ - {id: 8f7770b0d6d32ba1, ts: 598000000000, entity-tag: entity1}
+ - {id: 4d4a408373774341, ts: 599000000000, entity-tag: entity1}
+ - {id: ebf9605efc49854a, ts: 600000000000, entity-tag: entity1}
+ - {id: f4248811c87fec27, ts: 601000000000, entity-tag: entity1}
+ - {id: 3dd4d2710d0a7a3d, ts: 602000000000, entity-tag: entity1}
+ - {id: 8d81bf72b9949f6e, ts: 603000000000, entity-tag: entity1}
+ - {id: 9f327251973bf6ee, ts: 604000000000, entity-tag: entity1}
+ - {id: 392e80c63c0b440b, ts: 605000000000, entity-tag: entity1}
+ - {id: 329587c0d6583a64, ts: 606000000000, entity-tag: entity1}
+ - {id: 4d90ad3f712e6ec0, ts: 607000000000, entity-tag: entity1}
+ - {id: 5a5c7f6c9d4f9234, ts: 608000000000, entity-tag: entity1}
+ - {id: ad9f8474fdef4792, ts: 609000000000, entity-tag: entity1}
+ - {id: 30b7526a4e211dec, ts: 610000000000, entity-tag: entity1}
+ - {id: b967bf747215801d, ts: 611000000000, entity-tag: entity1}
+ - {id: e91054755c523aa7, ts: 612000000000, entity-tag: entity1}
+ - {id: bcd4e5295c7ab6c2, ts: 613000000000, entity-tag: entity1}
+ - {id: 6513a5d5730c8713, ts: 614000000000, entity-tag: entity1}
+ - {id: 83d3b83e488e610e, ts: 615000000000, entity-tag: entity1}
+ - {id: b926d93b90f09bec, ts: 616000000000, entity-tag: entity1}
+ - {id: cdfd0b938197cb7e, ts: 617000000000, entity-tag: entity1}
+ - {id: ec9e1a54eb7e95d2, ts: 618000000000, entity-tag: entity1}
+ - {id: 2acec5b03e4ce8d7, ts: 619000000000, entity-tag: entity1}
+ - {id: b26c07dc9f9a3875, ts: 620000000000, entity-tag: entity1}
+ - {id: ae05b68886b5cb28, ts: 621000000000, entity-tag: entity1}
+ - {id: 7d88b4e894046e7b, ts: 622000000000, entity-tag: entity1}
+ - {id: 8e0b5a7c0d258d0c, ts: 623000000000, entity-tag: entity1}
+ - {id: a7b654bbbc84b8d7, ts: 624000000000, entity-tag: entity1}
+ - {id: 255155c2b6855f25, ts: 625000000000, entity-tag: entity1}
+ - {id: a193c2ca98065847, ts: 626000000000, entity-tag: entity1}
+ - {id: 530ca95b44ed9e02, ts: 627000000000, entity-tag: entity1}
+ - {id: c40fb327055a4296, ts: 628000000000, entity-tag: entity1}
+ - {id: ef1bff5b52522077, ts: 629000000000, entity-tag: entity1}
+ - {id: 3308cd812843fd2a, ts: 630000000000, entity-tag: entity1}
+ - {id: c3fabd07639d37c6, ts: 631000000000, entity-tag: entity1}
+ - {id: 6b5a2e5dae8de838, ts: 632000000000, entity-tag: entity1}
+ - {id: 5e53c7d62f1bf2c8, ts: 633000000000, entity-tag: entity1}
+ - {id: 0d0f5ca1ba9ca719, ts: 634000000000, entity-tag: entity1}
+ - {id: ce4ed6133202db8a, ts: 635000000000, entity-tag: entity1}
+ - {id: 6adf519dda9f3c6b, ts: 636000000000, entity-tag: entity1}
+ - {id: e276f9fc71eaa5ea, ts: 637000000000, entity-tag: entity1}
+ - {id: 7983b9e10723bc9f, ts: 638000000000, entity-tag: entity1}
+ - {id: 7328109230b1ea1c, ts: 639000000000, entity-tag: entity1}
+ - {id: eb864b5917ec88eb, ts: 640000000000, entity-tag: entity1}
+ - {id: e9edf82de2e57b47, ts: 641000000000, entity-tag: entity1}
+ - {id: 0e9713f0d0597067, ts: 642000000000, entity-tag: entity1}
+ - {id: ba67909d6f0aa252, ts: 643000000000, entity-tag: entity1}
+ - {id: 9748e31a64d2a06e, ts: 644000000000, entity-tag: entity1}
+ - {id: 07a90182a1e6deed, ts: 645000000000, entity-tag: entity1}
+ - {id: 30a8886ae75c94ef, ts: 646000000000, entity-tag: entity1}
+ - {id: 0164fac1e1204671, ts: 647000000000, entity-tag: entity1}
+ - {id: 23655405a40c36e6, ts: 648000000000, entity-tag: entity1}
+ - {id: 28eed47fd86ea07e, ts: 649000000000, entity-tag: entity1}
+ - {id: bd4408e3bc44a886, ts: 650000000000, entity-tag: entity1}
+ - {id: 3008b6ee81ec305f, ts: 651000000000, entity-tag: entity1}
+ - {id: 4ba6f7d19868c617, ts: 652000000000, entity-tag: entity1}
+ - {id: 0fc078aced6c03e8, ts: 653000000000, entity-tag: entity1}
+ - {id: 88a8868785b1978d, ts: 654000000000, entity-tag: entity1}
+ - {id: dced738953a7be9e, ts: 655000000000, entity-tag: entity1}
+ - {id: a5a0f9ab4043964f, ts: 656000000000, entity-tag: entity1}
+ - {id: a10b3c3a13c5e40d, ts: 657000000000, entity-tag: entity1}
+ - {id: a34ea49aa5f0fbd0, ts: 658000000000, entity-tag: entity1}
+ - {id: 3bf29b16e903168b, ts: 659000000000, entity-tag: entity1}
+ - {id: 5aea1aa2c75f9ca6, ts: 660000000000, entity-tag: entity1}
+ - {id: 8644dd1cbd240dd0, ts: 661000000000, entity-tag: entity1}
+ - {id: 691fe3eab7c123f0, ts: 662000000000, entity-tag: entity1}
+ - {id: 380d77404b79238b, ts: 663000000000, entity-tag: entity1}
+ - {id: 8b1b316cf86e078f, ts: 664000000000, entity-tag: entity1}
+ - {id: 095f667cdc2b09fa, ts: 665000000000, entity-tag: entity1}
+ - {id: a781fb1e53a3c21c, ts: 666000000000, entity-tag: entity1}
+ - {id: 561e8caf874590ad, ts: 667000000000, entity-tag: entity1}
+ - {id: 2c876a559542ce67, ts: 668000000000, entity-tag: entity1}
+ - {id: c56a5f955e714253, ts: 669000000000, entity-tag: entity1}
+ - {id: c63b0a4463e14c9c, ts: 670000000000, entity-tag: entity1}
+ - {id: 68ca0f94c3786cf4, ts: 671000000000, entity-tag: entity1}
+ - {id: 7214db7a79c0868e, ts: 672000000000, entity-tag: entity1}
+ - {id: a0dd159b53158ffd, ts: 673000000000, entity-tag: entity1}
+ - {id: 8d800299817b7ed3, ts: 674000000000, entity-tag: entity1}
+ - {id: d800e7d4bb77a931, ts: 675000000000, entity-tag: entity1}
+ - {id: 3fd0d890b7b63ebb, ts: 676000000000, entity-tag: entity1}
+ - {id: 8e3a1b1dd776cf28, ts: 677000000000, entity-tag: entity1}
+ - {id: 396da1a6cf2f08e3, ts: 678000000000, entity-tag: entity1}
+ - {id: eb4dde416cb629ae, ts: 679000000000, entity-tag: entity1}
+ - {id: db5f06fb8b3ea5c0, ts: 680000000000, entity-tag: entity1}
+ - {id: 385c769807e0edbc, ts: 681000000000, entity-tag: entity1}
+ - {id: 936805fc0e3b884a, ts: 682000000000, entity-tag: entity1}
+ - {id: 5948cdea886dcc6c, ts: 683000000000, entity-tag: entity1}
+ - {id: 49b1e0142ca4cbc2, ts: 684000000000, entity-tag: entity1}
+ - {id: 3aea6ac32e47d59f, ts: 685000000000, entity-tag: entity1}
+ - {id: 5301fce5cf54b0fa, ts: 686000000000, entity-tag: entity1}
+ - {id: 90bc580923f21cd0, ts: 687000000000, entity-tag: entity1}
+ - {id: 37d2a907db4f84a3, ts: 688000000000, entity-tag: entity1}
+ - {id: ca91e0424aa501a4, ts: 689000000000, entity-tag: entity1}
+ - {id: c2bc77af4e939eed, ts: 690000000000, entity-tag: entity1}
+ - {id: 14449b8936d48b70, ts: 691000000000, entity-tag: entity1}
+ - {id: 5a071d6b5d96ded0, ts: 692000000000, entity-tag: entity1}
+ - {id: 7ddfe6e3232c20ab, ts: 693000000000, entity-tag: entity1}
+ - {id: 6d4227da583885a0, ts: 694000000000, entity-tag: entity1}
+ - {id: a9309d59cf54699b, ts: 695000000000, entity-tag: entity1}
+ - {id: 5695ad36812ed381, ts: 696000000000, entity-tag: entity1}
+ - {id: f3ec8690fb39f99d, ts: 697000000000, entity-tag: entity1}
+ - {id: fde732a0d634a0a9, ts: 698000000000, entity-tag: entity1}
+ - {id: 86f9d2dc1aeea87d, ts: 699000000000, entity-tag: entity1}
+ - {id: bd1cd9832d91fceb, ts: 700000000000, entity-tag: entity1}
+ - {id: ec232f809f47c6e8, ts: 701000000000, entity-tag: entity1}
+ - {id: db77de533bfd0b8a, ts: 702000000000, entity-tag: entity1}
+ - {id: b0438c0cf06d236f, ts: 703000000000, entity-tag: entity1}
+ - {id: 7d4824c9ee557eca, ts: 704000000000, entity-tag: entity1}
+ - {id: 534d1c4b45efb698, ts: 705000000000, entity-tag: entity1}
+ - {id: 7cab524f26470aa2, ts: 706000000000, entity-tag: entity1}
+ - {id: a90c77284b36833d, ts: 707000000000, entity-tag: entity1}
+ - {id: d5321ac7c26af32d, ts: 708000000000, entity-tag: entity1}
+ - {id: eb76238c36ea400c, ts: 709000000000, entity-tag: entity1}
+ - {id: 2cb281b4c0a51560, ts: 710000000000, entity-tag: entity1}
+ - {id: ea90e499369f8281, ts: 711000000000, entity-tag: entity1}
+ - {id: b2c2d7e565c58eea, ts: 712000000000, entity-tag: entity1}
+ - {id: f904abee53703c11, ts: 713000000000, entity-tag: entity1}
+ - {id: 87a9e2b3f5d7e8d3, ts: 714000000000, entity-tag: entity1}
+ - {id: 1c65bc5d455c0619, ts: 715000000000, entity-tag: entity1}
+ - {id: a4604e65f3677a90, ts: 716000000000, entity-tag: entity1}
+ - {id: b00d5ae448d488eb, ts: 717000000000, entity-tag: entity1}
+ - {id: 269cdf1d89c41dce, ts: 718000000000, entity-tag: entity1}
+ - {id: c76a99a6de901e68, ts: 719000000000, entity-tag: entity1}
+ - {id: f97fa9aa3f71fb38, ts: 720000000000, entity-tag: entity1}
+ - {id: c45f1db881d2a595, ts: 721000000000, entity-tag: entity1}
+ - {id: 0c5ba5637cf6b1ca, ts: 722000000000, entity-tag: entity1}
+ - {id: 432788864ef5ffaf, ts: 723000000000, entity-tag: entity1}
+ - {id: 749cf1e068c038e9, ts: 724000000000, entity-tag: entity1}
+ - {id: 9463dd3aee0f9f9e, ts: 725000000000, entity-tag: entity1}
+ - {id: aad83ba98b9f3353, ts: 726000000000, entity-tag: entity1}
+ - {id: a76729dc16e7e996, ts: 727000000000, entity-tag: entity1}
+ - {id: 6c1543287bc9552a, ts: 728000000000, entity-tag: entity1}
+ - {id: 07f83afdb44353ac, ts: 729000000000, entity-tag: entity1}
+ - {id: c13afd020d5ca024, ts: 730000000000, entity-tag: entity1}
+ - {id: 5080b749eb99f912, ts: 731000000000, entity-tag: entity1}
+ - {id: 8f14983906e9aaa5, ts: 732000000000, entity-tag: entity1}
+ - {id: d00546ee90c89a45, ts: 733000000000, entity-tag: entity1}
+ - {id: da5a2b86359935f9, ts: 734000000000, entity-tag: entity1}
+ - {id: 9e76c256cd15bf10, ts: 735000000000, entity-tag: entity1}
+ - {id: f5aad2acc38ed012, ts: 736000000000, entity-tag: entity1}
+ - {id: 56d03e963bab5059, ts: 737000000000, entity-tag: entity1}
+ - {id: fa18c771823f6c1f, ts: 738000000000, entity-tag: entity1}
+ - {id: d3bbe69a77162966, ts: 739000000000, entity-tag: entity1}
+ - {id: 2b91a0b523994e64, ts: 740000000000, entity-tag: entity1}
+ - {id: 597ef00f974ce250, ts: 741000000000, entity-tag: entity1}
+ - {id: 4af4550d1da6d059, ts: 742000000000, entity-tag: entity1}
+ - {id: f5921e28a6996b4c, ts: 743000000000, entity-tag: entity1}
+ - {id: b918842c1112492d, ts: 744000000000, entity-tag: entity1}
+ - {id: 28f742c6cfcb981f, ts: 745000000000, entity-tag: entity1}
+ - {id: afc8d10a4e55facc, ts: 746000000000, entity-tag: entity1}
+ - {id: bf0e6950d97250e7, ts: 747000000000, entity-tag: entity1}
+ - {id: 2c8584f970735e5b, ts: 748000000000, entity-tag: entity1}
+ - {id: c92ea5794014911b, ts: 749000000000, entity-tag: entity1}
+ - {id: 7368995f35aa6bf2, ts: 750000000000, entity-tag: entity1}
+ - {id: 63d50e70fd3985af, ts: 751000000000, entity-tag: entity1}
+ - {id: dd0460eac4dac2e4, ts: 752000000000, entity-tag: entity1}
+ - {id: ed8e3ffe5fb03e38, ts: 753000000000, entity-tag: entity1}
+ - {id: 49ecbcc09e0f4a57, ts: 754000000000, entity-tag: entity1}
+ - {id: e2e9f19196fdaf56, ts: 755000000000, entity-tag: entity1}
+ - {id: 16a7e26d9d50706e, ts: 756000000000, entity-tag: entity1}
+ - {id: 711658143ae8574f, ts: 757000000000, entity-tag: entity1}
+ - {id: ea3ef484e989010b, ts: 758000000000, entity-tag: entity1}
+ - {id: c68a9311ecf6edd0, ts: 759000000000, entity-tag: entity1}
+ - {id: c5b2db48c8fd32ff, ts: 760000000000, entity-tag: entity1}
+ - {id: 2e5940566c183546, ts: 761000000000, entity-tag: entity1}
+ - {id: 0671342196ed6ead, ts: 762000000000, entity-tag: entity1}
+ - {id: d1479069a8526761, ts: 763000000000, entity-tag: entity1}
+ - {id: cf7ae1435b544259, ts: 764000000000, entity-tag: entity1}
+ - {id: 3876ceefada8106c, ts: 765000000000, entity-tag: entity1}
+ - {id: f32fd5c596f1dff7, ts: 766000000000, entity-tag: entity1}
+ - {id: 9ad7622204c2cc2b, ts: 767000000000, entity-tag: entity1}
+ - {id: ab5bbb4dfcb31774, ts: 768000000000, entity-tag: entity1}
+ - {id: 0d51299be74b0c26, ts: 769000000000, entity-tag: entity1}
+ - {id: 50f9836591bdce11, ts: 770000000000, entity-tag: entity1}
+ - {id: a9d70277ed359614, ts: 771000000000, entity-tag: entity1}
+ - {id: 4965d29e6f941827, ts: 772000000000, entity-tag: entity1}
+ - {id: c208da8fb316dfec, ts: 773000000000, entity-tag: entity1}
+ - {id: fc20e0d41d9e5e38, ts: 774000000000, entity-tag: entity1}
+ - {id: e19a50147a0e63eb, ts: 775000000000, entity-tag: entity1}
+ - {id: 7616275fe61fae8c, ts: 776000000000, entity-tag: entity1}
+ - {id: 1cd246a781649f5c, ts: 777000000000, entity-tag: entity1}
+ - {id: e8e003a7b39a11eb, ts: 778000000000, entity-tag: entity1}
+ - {id: d280e660100926ce, ts: 779000000000, entity-tag: entity1}
+ - {id: a55c73dd3221a1f6, ts: 780000000000, entity-tag: entity1}
+ - {id: 727547070a3334d4, ts: 781000000000, entity-tag: entity1}
+ - {id: f3f2c18cd8ac6f46, ts: 782000000000, entity-tag: entity1}
+ - {id: 81f04309d01aea0f, ts: 783000000000, entity-tag: entity1}
+ - {id: 1b23593523afb1b4, ts: 784000000000, entity-tag: entity1}
+ - {id: fd75a855c760a57f, ts: 785000000000, entity-tag: entity1}
+ - {id: aa2e7dd456f3f850, ts: 786000000000, entity-tag: entity1}
+ - {id: 1305cc98a8813a1e, ts: 787000000000, entity-tag: entity1}
+ - {id: 9b125aabef8b5c2d, ts: 788000000000, entity-tag: entity1}
+ - {id: fab5b6e3478f6d6b, ts: 789000000000, entity-tag: entity1}
+ - {id: 7800ef356f67fd3b, ts: 790000000000, entity-tag: entity1}
+ - {id: 283a239138780da8, ts: 791000000000, entity-tag: entity1}
+ - {id: 8a3b444450d4720a, ts: 792000000000, entity-tag: entity1}
+ - {id: 22333711f780cb7e, ts: 793000000000, entity-tag: entity1}
+ - {id: 87d56e00a88c2e95, ts: 794000000000, entity-tag: entity1}
+ - {id: 79ab9aa07c87ae99, ts: 795000000000, entity-tag: entity1}
+ - {id: 2d7533f7b5bb0fad, ts: 796000000000, entity-tag: entity1}
+ - {id: dec610e14075f10b, ts: 797000000000, entity-tag: entity1}
+ - {id: d52952a64110bc29, ts: 798000000000, entity-tag: entity1}
+ - {id: 9376b72f7b1e8bf8, ts: 799000000000, entity-tag: entity1}
+ - {id: 368abe7d9a2aebe2, ts: 800000000000, entity-tag: entity1}
+ - {id: 824a00a52fc6234f, ts: 801000000000, entity-tag: entity1}
+ - {id: 5f3055bb4f94b325, ts: 802000000000, entity-tag: entity1}
+ - {id: a9a0eca5b163805a, ts: 803000000000, entity-tag: entity1}
+ - {id: a8c95adb6ca67ba9, ts: 804000000000, entity-tag: entity1}
+ - {id: 692e1ab80950d010, ts: 805000000000, entity-tag: entity1}
+ - {id: 597acc26b13749e4, ts: 806000000000, entity-tag: entity1}
+ - {id: c60b4cf69be1ee9e, ts: 807000000000, entity-tag: entity1}
+ - {id: a7c8290fa6176d0e, ts: 808000000000, entity-tag: entity1}
+ - {id: 324b4613c41f551b, ts: 809000000000, entity-tag: entity1}
+ - {id: 994e3effef7764a6, ts: 810000000000, entity-tag: entity1}
+ - {id: 179a9085f8417d2e, ts: 811000000000, entity-tag: entity1}
+ - {id: 8e8e2c9525a64b2d, ts: 812000000000, entity-tag: entity1}
+ - {id: 03c8bb02185261fd, ts: 813000000000, entity-tag: entity1}
+ - {id: 8fbb0ca870080c0d, ts: 814000000000, entity-tag: entity1}
+ - {id: 065be705aa6c2966, ts: 815000000000, entity-tag: entity1}
+ - {id: 8d3cd9ce66e478cb, ts: 816000000000, entity-tag: entity1}
+ - {id: 67330e39cd534866, ts: 817000000000, entity-tag: entity1}
+ - {id: 8654550f34d7e5f9, ts: 818000000000, entity-tag: entity1}
+ - {id: a7c1eca52807bab7, ts: 819000000000, entity-tag: entity1}
+ - {id: 1dd04303f985618b, ts: 820000000000, entity-tag: entity1}
+ - {id: 650fc7953b60a4b2, ts: 821000000000, entity-tag: entity1}
+ - {id: 64dc45b89fcabe51, ts: 822000000000, entity-tag: entity1}
+ - {id: 98aa4005dd55cbf4, ts: 823000000000, entity-tag: entity1}
+ - {id: 4190a689bed6a88c, ts: 824000000000, entity-tag: entity1}
+ - {id: ab593cc45e41ac8f, ts: 825000000000, entity-tag: entity1}
+ - {id: c53d4a6aca70c60b, ts: 826000000000, entity-tag: entity1}
+ - {id: 3011bf5dfbb2ab3a, ts: 827000000000, entity-tag: entity1}
+ - {id: d76eeaf3a4223e6a, ts: 828000000000, entity-tag: entity1}
+ - {id: 478d941a3658e34d, ts: 829000000000, entity-tag: entity1}
+ - {id: 909a62127cc5068a, ts: 830000000000, entity-tag: entity1}
+ - {id: f27aa0c15f77a511, ts: 831000000000, entity-tag: entity1}
+ - {id: bf289fe019883067, ts: 832000000000, entity-tag: entity1}
+ - {id: cc19e5011f7dc43d, ts: 833000000000, entity-tag: entity1}
+ - {id: 353516e5a7c64d80, ts: 834000000000, entity-tag: entity1}
+ - {id: 602859b38311dd74, ts: 835000000000, entity-tag: entity1}
+ - {id: a43319a37f79af51, ts: 836000000000, entity-tag: entity1}
+ - {id: 87cd4dad2481d3f1, ts: 837000000000, entity-tag: entity1}
+ - {id: ef65eb9bed568397, ts: 838000000000, entity-tag: entity1}
+ - {id: 06fc9591a09aa61b, ts: 839000000000, entity-tag: entity1}
+ - {id: 39ce55e5ae922913, ts: 840000000000, entity-tag: entity1}
+ - {id: bb0525a54dcbde17, ts: 841000000000, entity-tag: entity1}
+ - {id: 76793a4115bc0ce9, ts: 842000000000, entity-tag: entity1}
+ - {id: 86464a116c3299a2, ts: 843000000000, entity-tag: entity1}
+ - {id: d44dacc9c50a74e4, ts: 844000000000, entity-tag: entity1}
+ - {id: e93bd3b4fd3962e3, ts: 845000000000, entity-tag: entity1}
+ - {id: 80c4eb6649d2119a, ts: 846000000000, entity-tag: entity1}
+ - {id: 268dedc91281147f, ts: 847000000000, entity-tag: entity1}
+ - {id: 60b0f33900c73758, ts: 848000000000, entity-tag: entity1}
+ - {id: e2a6a511a239910d, ts: 849000000000, entity-tag: entity1}
+ - {id: 58b1d69b280f7479, ts: 850000000000, entity-tag: entity1}
+ - {id: 26b109c51e119e65, ts: 851000000000, entity-tag: entity1}
+ - {id: 0fa76578de67657a, ts: 852000000000, entity-tag: entity1}
+ - {id: a257aa682b137ee0, ts: 853000000000, entity-tag: entity1}
+ - {id: f93220787f443661, ts: 854000000000, entity-tag: entity1}
+ - {id: 032c4d31ae7aaf80, ts: 855000000000, entity-tag: entity1}
+ - {id: 5eaca94c68224a2e, ts: 856000000000, entity-tag: entity1}
+ - {id: 6646225b88ff9cad, ts: 857000000000, entity-tag: entity1}
+ - {id: 2f2a3f8b84b93bf2, ts: 858000000000, entity-tag: entity1}
+ - {id: c4e8744e16a08578, ts: 859000000000, entity-tag: entity1}
+ - {id: 3bce116bb8e2b94c, ts: 860000000000, entity-tag: entity1}
+ - {id: 5c5e15b4fef2f4cb, ts: 861000000000, entity-tag: entity1}
+ - {id: 19d1fc96fbed8f40, ts: 862000000000, entity-tag: entity1}
+ - {id: 79ed7ea026a9e972, ts: 863000000000, entity-tag: entity1}
+ - {id: 226ba9fadd5f2875, ts: 864000000000, entity-tag: entity1}
+ - {id: 80880ff4d6edd5f7, ts: 865000000000, entity-tag: entity1}
+ - {id: c0cf1b85bad89d65, ts: 866000000000, entity-tag: entity1}
+ - {id: e02d891ac3cc1715, ts: 867000000000, entity-tag: entity1}
+ - {id: 41e0d9962e37dc5e, ts: 868000000000, entity-tag: entity1}
+ - {id: 03e6c27c659ff156, ts: 869000000000, entity-tag: entity1}
+ - {id: a9301a558fd21aee, ts: 870000000000, entity-tag: entity1}
+ - {id: 83b68d92d0509136, ts: 871000000000, entity-tag: entity1}
+ - {id: c5e7b8d9947a89f1, ts: 872000000000, entity-tag: entity1}
+ - {id: c31a77ada65b1176, ts: 873000000000, entity-tag: entity1}
+ - {id: 53e4b867c646196c, ts: 874000000000, entity-tag: entity1}
+ - {id: fb0da09df5d90197, ts: 875000000000, entity-tag: entity1}
+ - {id: 634857c0e4079a23, ts: 876000000000, entity-tag: entity1}
+ - {id: a73dc3d87437e5e2, ts: 877000000000, entity-tag: entity1}
+ - {id: d2e5848ce963fb69, ts: 878000000000, entity-tag: entity1}
+ - {id: 3e205a51f6658158, ts: 879000000000, entity-tag: entity1}
+ - {id: 7ac87f5b3af5f2f3, ts: 880000000000, entity-tag: entity1}
+ - {id: ef9a3ef1f2b46fb0, ts: 881000000000, entity-tag: entity1}
+ - {id: 318d29550ccbb824, ts: 882000000000, entity-tag: entity1}
+ - {id: 7110251226155436, ts: 883000000000, entity-tag: entity1}
+ - {id: e0d2ec1d37ebfd85, ts: 884000000000, entity-tag: entity1}
+ - {id: d0ed9886a5f9b500, ts: 885000000000, entity-tag: entity1}
+ - {id: a3de801e544b16b1, ts: 886000000000, entity-tag: entity1}
+ - {id: c856a36172ebec27, ts: 887000000000, entity-tag: entity1}
+ - {id: 012963278133a125, ts: 888000000000, entity-tag: entity1}
+ - {id: 0cbea9cff1a8667b, ts: 889000000000, entity-tag: entity1}
+ - {id: f93ea3cebfa77d15, ts: 890000000000, entity-tag: entity1}
+ - {id: 271107b24317176c, ts: 891000000000, entity-tag: entity1}
+ - {id: 2034ff94b0aa3c55, ts: 892000000000, entity-tag: entity1}
+ - {id: 2588cbe834f9fa14, ts: 893000000000, entity-tag: entity1}
+ - {id: 2bf2204189e81e77, ts: 894000000000, entity-tag: entity1}
+ - {id: 9c4b2d6327648cd5, ts: 895000000000, entity-tag: entity1}
+ - {id: 48cfd5dc004483e8, ts: 896000000000, entity-tag: entity1}
+ - {id: 933763a8030f1700, ts: 897000000000, entity-tag: entity1}
+ - {id: c04c0c764f55958f, ts: 898000000000, entity-tag: entity1}
+ - {id: a8460db0eaf94044, ts: 899000000000, entity-tag: entity1}
+ - {id: 940dfafd3bdc2823, ts: 900000000000, entity-tag: entity1}
+ - {id: 67a824c5e8a75f82, ts: 901000000000, entity-tag: entity1}
+ - {id: 6617c48c507e775f, ts: 902000000000, entity-tag: entity1}
+ - {id: 3613181e3f53dc58, ts: 903000000000, entity-tag: entity1}
+ - {id: c7ea3cd68fded74a, ts: 904000000000, entity-tag: entity1}
+ - {id: 5169c728dc5186df, ts: 905000000000, entity-tag: entity1}
+ - {id: a456d82f7e6cf498, ts: 906000000000, entity-tag: entity1}
+ - {id: 814559b877bb1fa9, ts: 907000000000, entity-tag: entity1}
+ - {id: 1af7120ee16115d2, ts: 908000000000, entity-tag: entity1}
+ - {id: 96891065706f1795, ts: 909000000000, entity-tag: entity1}
+ - {id: b3be1aac1bb02ad4, ts: 910000000000, entity-tag: entity1}
+ - {id: 959c55f242097553, ts: 911000000000, entity-tag: entity1}
+ - {id: 7f19bc9c3c3c6058, ts: 912000000000, entity-tag: entity1}
+ - {id: 48f7a9b923be9c82, ts: 913000000000, entity-tag: entity1}
+ - {id: dacc242fd0172629, ts: 914000000000, entity-tag: entity1}
+ - {id: 5762849e5046c6fd, ts: 915000000000, entity-tag: entity1}
+ - {id: 77ee93703df5cc49, ts: 916000000000, entity-tag: entity1}
+ - {id: 1612240bca436f2c, ts: 917000000000, entity-tag: entity1}
+ - {id: fa4fb7a138c1e5d0, ts: 918000000000, entity-tag: entity1}
+ - {id: c41d222c44ce3ceb, ts: 919000000000, entity-tag: entity1}
+ - {id: 59bb97f552c58573, ts: 920000000000, entity-tag: entity1}
+ - {id: eb6d65e1c28b6385, ts: 921000000000, entity-tag: entity1}
+ - {id: 89ba5657aeb47432, ts: 922000000000, entity-tag: entity1}
+ - {id: f4ac403d391bef00, ts: 923000000000, entity-tag: entity1}
+ - {id: 8c52443fc53fc013, ts: 924000000000, entity-tag: entity1}
+ - {id: 09e24bd95407cbba, ts: 925000000000, entity-tag: entity1}
+ - {id: 5100adbb71b54f6b, ts: 926000000000, entity-tag: entity1}
+ - {id: aecdafaf625a4c6b, ts: 927000000000, entity-tag: entity1}
+ - {id: ac178c0669fad0cc, ts: 928000000000, entity-tag: entity1}
+ - {id: 06f6210f217d97f3, ts: 929000000000, entity-tag: entity1}
+ - {id: 5b306a27c8b0d7af, ts: 930000000000, entity-tag: entity1}
+ - {id: 45b54cf07401f8fb, ts: 931000000000, entity-tag: entity1}
+ - {id: e9b1b13dfd4aaf4b, ts: 932000000000, entity-tag: entity1}
+ - {id: 51a9b350ecb0b42a, ts: 933000000000, entity-tag: entity1}
+ - {id: bb1bf164a4525431, ts: 934000000000, entity-tag: entity1}
+ - {id: 6671752c007a6e7a, ts: 935000000000, entity-tag: entity1}
+ - {id: f487e8e05f3292c2, ts: 936000000000, entity-tag: entity1}
+ - {id: 027aa78e4e6febde, ts: 937000000000, entity-tag: entity1}
+ - {id: 6e1c39ab2e460d1b, ts: 938000000000, entity-tag: entity1}
+ - {id: 75136ad657c22377, ts: 939000000000, entity-tag: entity1}
+ - {id: 1e224229856317ec, ts: 940000000000, entity-tag: entity1}
+ - {id: b403336f70593342, ts: 941000000000, entity-tag: entity1}
+ - {id: 4393943882416089, ts: 942000000000, entity-tag: entity1}
+ - {id: 55d96a4057f50f9c, ts: 943000000000, entity-tag: entity1}
+ - {id: 8acfe3377434764c, ts: 944000000000, entity-tag: entity1}
+ - {id: c7576f9f92897cbe, ts: 945000000000, entity-tag: entity1}
+ - {id: d656aef33d0d886b, ts: 946000000000, entity-tag: entity1}
+ - {id: 434d2593309f9b61, ts: 947000000000, entity-tag: entity1}
+ - {id: 17a5c192482f4698, ts: 948000000000, entity-tag: entity1}
+ - {id: c787acec0b96d9b3, ts: 949000000000, entity-tag: entity1}
+ - {id: c9dd10f59767db83, ts: 950000000000, entity-tag: entity1}
+ - {id: 3a169071414b3a8d, ts: 951000000000, entity-tag: entity1}
+ - {id: b713f170572273df, ts: 952000000000, entity-tag: entity1}
+ - {id: 3f6fabd3de84a9ca, ts: 953000000000, entity-tag: entity1}
+ - {id: 1228b040c58e3008, ts: 954000000000, entity-tag: entity1}
+ - {id: f31bde26cb3a9983, ts: 955000000000, entity-tag: entity1}
+ - {id: 65aff27c8fa3ecbf, ts: 956000000000, entity-tag: entity1}
+ - {id: 25cf4f13b323edd5, ts: 957000000000, entity-tag: entity1}
+ - {id: 32b79843c1097b7b, ts: 958000000000, entity-tag: entity1}
+ - {id: da5d3b8210c338f6, ts: 959000000000, entity-tag: entity1}
+ - {id: cfaf4b2878e3fd29, ts: 960000000000, entity-tag: entity1}
+ - {id: 8e3fcc8abb57e24a, ts: 961000000000, entity-tag: entity1}
+ - {id: c433b245cab2ebac, ts: 962000000000, entity-tag: entity1}
+ - {id: 2a2a9ae41c72501b, ts: 963000000000, entity-tag: entity1}
+ - {id: 59af14543f3760a2, ts: 964000000000, entity-tag: entity1}
+ - {id: 00c9022b1020ee8b, ts: 965000000000, entity-tag: entity1}
+ - {id: 06f3bf23aa6ff7bb, ts: 966000000000, entity-tag: entity1}
+ - {id: fc213a1a434c22a7, ts: 967000000000, entity-tag: entity1}
+ - {id: f1b3c3d4795bbf21, ts: 968000000000, entity-tag: entity1}
+ - {id: 89261b79b1522b10, ts: 969000000000, entity-tag: entity1}
+ - {id: 98318527c9565c41, ts: 970000000000, entity-tag: entity1}
+ - {id: f1f8ef7ac42a58b1, ts: 971000000000, entity-tag: entity1}
+ - {id: 3d7a7cb295602676, ts: 972000000000, entity-tag: entity1}
+ - {id: 06ef8f6ffea7230e, ts: 973000000000, entity-tag: entity1}
+ - {id: 01bb9fbc633345b9, ts: 974000000000, entity-tag: entity1}
+ - {id: 630dc9e908045e72, ts: 975000000000, entity-tag: entity1}
+ - {id: 8eb6dad2891de625, ts: 976000000000, entity-tag: entity1}
+ - {id: d5269cf17f170acd, ts: 977000000000, entity-tag: entity1}
+ - {id: 13f874b396834994, ts: 978000000000, entity-tag: entity1}
+ - {id: 753cf94a6c0988e4, ts: 979000000000, entity-tag: entity1}
+ - {id: 67164b414943740d, ts: 980000000000, entity-tag: entity1}
+ - {id: 2e372abfd79bbc19, ts: 981000000000, entity-tag: entity1}
+ - {id: 30bd2b2fca79b652, ts: 982000000000, entity-tag: entity1}
+ - {id: 6b2c21d836ab15d1, ts: 983000000000, entity-tag: entity1}
+ - {id: ca62267bc909767c, ts: 984000000000, entity-tag: entity1}
+ - {id: 0f9791e21e5fadf5, ts: 985000000000, entity-tag: entity1}
+ - {id: 3b580899ea2b436e, ts: 986000000000, entity-tag: entity1}
+ - {id: 3f9ca3c8f6a1c44d, ts: 987000000000, entity-tag: entity1}
+ - {id: 2280571863bce00d, ts: 988000000000, entity-tag: entity1}
+ - {id: 506b35d78dc310c5, ts: 989000000000, entity-tag: entity1}
+ - {id: be7633971fb7cda0, ts: 990000000000, entity-tag: entity1}
+ - {id: 92cfd8160ea7dc1c, ts: 991000000000, entity-tag: entity1}
+ - {id: eba6c72d27c81b39, ts: 992000000000, entity-tag: entity1}
+ - {id: c122c4534791aab7, ts: 993000000000, entity-tag: entity1}
+ - {id: 32b370710cdab97a, ts: 994000000000, entity-tag: entity1}
+ - {id: c54c9be98db95f8d, ts: 995000000000, entity-tag: entity1}
+ - {id: e70a8e57f33902f0, ts: 996000000000, entity-tag: entity1}
+ - {id: 4de0f3131f7ab30c, ts: 997000000000, entity-tag: entity1}
+ - {id: 7d5d16ab0ef1306d, ts: 998000000000, entity-tag: entity1}
+ - {id: c0b2a06785ecbb12, ts: 999000000000, entity-tag: entity1}
+ - {id: 2f437ac548637970, ts: 1000000000000, entity-tag: entity1}
+ - {id: 16e206b8cb4c0fa5, ts: 1001000000000, entity-tag: entity1}
+ - {id: 206181e39d6bd5c1, ts: 1002000000000, entity-tag: entity1}
+ - {id: 741d7d8dcc065ec4, ts: 1003000000000, entity-tag: entity1}
+ - {id: 49029f22b5578da6, ts: 1004000000000, entity-tag: entity1}
+ - {id: 814d12064d6dbfe2, ts: 1005000000000, entity-tag: entity1}
+ - {id: 13282945b8a7716b, ts: 1006000000000, entity-tag: entity1}
+ - {id: eb0fe2a9fd0b978a, ts: 1007000000000, entity-tag: entity1}
+ - {id: e760202720a8b991, ts: 1008000000000, entity-tag: entity1}
+ - {id: f88f7cc427a38d70, ts: 1009000000000, entity-tag: entity1}
+ - {id: ed97b2108235222f, ts: 1010000000000, entity-tag: entity1}
+ - {id: 8ea0a5d2c11eb52b, ts: 1011000000000, entity-tag: entity1}
+ - {id: febd5d985f5a6ca5, ts: 1012000000000, entity-tag: entity1}
+ - {id: fa8400e2e5c778b3, ts: 1013000000000, entity-tag: entity1}
+ - {id: 4040493abbd20e7a, ts: 1014000000000, entity-tag: entity1}
+ - {id: d5f0284a858e94d9, ts: 1015000000000, entity-tag: entity1}
+ - {id: b696f94df26eb22e, ts: 1016000000000, entity-tag: entity1}
+ - {id: 9bcbe16710698f1e, ts: 1017000000000, entity-tag: entity1}
+ - {id: 56bd3685ea68115e, ts: 1018000000000, entity-tag: entity1}
+ - {id: 6dba507ccf28edb1, ts: 1019000000000, entity-tag: entity1}
+ - {id: 3f53bdf494cf61cf, ts: 1020000000000, entity-tag: entity1}
+ - {id: da0e7223bf35a355, ts: 1021000000000, entity-tag: entity1}
+ - {id: b039a1950ab018ae, ts: 1022000000000, entity-tag: entity1}
+ - {id: b7fd55b91400fab2, ts: 1023000000000, entity-tag: entity1}
+ - {id: 0efa07cbe58df9d7, ts: 1024000000000, entity-tag: entity1}
+ - {id: 89cfc4c608090f7d, ts: 1025000000000, entity-tag: entity1}
+ - {id: af78f245e0244787, ts: 1026000000000, entity-tag: entity1}
+ - {id: cdc6bbfd068d382d, ts: 1027000000000, entity-tag: entity1}
+ - {id: 365170070909e25f, ts: 1028000000000, entity-tag: entity1}
+ - {id: 122c458b737dffe7, ts: 1029000000000, entity-tag: entity1}
+ - {id: 188b6d3a27407232, ts: 1030000000000, entity-tag: entity1}
+ - {id: b40c6df4a5026eb8, ts: 1031000000000, entity-tag: entity1}
+ - {id: b3fe078877d856f4, ts: 1032000000000, entity-tag: entity1}
+ - {id: a16d3836c21bd5e4, ts: 1033000000000, entity-tag: entity1}
+ - {id: 59bed511f26ac3d9, ts: 1034000000000, entity-tag: entity1}
+ - {id: 909ad140d0b6d15b, ts: 1035000000000, entity-tag: entity1}
+ - {id: 787471f1d20f4077, ts: 1036000000000, entity-tag: entity1}
+ - {id: c2570695ea72cfec, ts: 1037000000000, entity-tag: entity1}
+ - {id: c6f985011c5ea38d, ts: 1038000000000, entity-tag: entity1}
+ - {id: 0d7c6411d31f6091, ts: 1039000000000, entity-tag: entity1}
+ - {id: 5203e045fe270dd4, ts: 1040000000000, entity-tag: entity1}
+ - {id: 539c8a92243dab5b, ts: 1041000000000, entity-tag: entity1}
+ - {id: 08b88c6dd9ecb857, ts: 1042000000000, entity-tag: entity1}
+ - {id: 005ac08485ec06da, ts: 1043000000000, entity-tag: entity1}
+ - {id: f2c096817006a5bc, ts: 1044000000000, entity-tag: entity1}
+ - {id: 263fe89601bd4ce5, ts: 1045000000000, entity-tag: entity1}
+ - {id: 2fbcb0936384f647, ts: 1046000000000, entity-tag: entity1}
+ - {id: 056ef1a5fa0e8065, ts: 1047000000000, entity-tag: entity1}
+ - {id: 89f2ee0589ca12ca, ts: 1048000000000, entity-tag: entity1}
+ - {id: efc6ba9777534105, ts: 1049000000000, entity-tag: entity1}
+ - {id: 505e344eadfafa5b, ts: 1050000000000, entity-tag: entity1}
+ - {id: b43fe560cf67d4d9, ts: 1051000000000, entity-tag: entity1}
+ - {id: cf00928e6500b1d3, ts: 1052000000000, entity-tag: entity1}
+ - {id: 940d1660b03ae47d, ts: 1053000000000, entity-tag: entity1}
+ - {id: e06a78e0bd7b25dc, ts: 1054000000000, entity-tag: entity1}
+ - {id: 5be0fd0968580380, ts: 1055000000000, entity-tag: entity1}
+ - {id: ec6c2bdbaa114066, ts: 1056000000000, entity-tag: entity1}
+ - {id: 9d68cbeaf28b58ac, ts: 1057000000000, entity-tag: entity1}
+ - {id: 2c581220a81c2ee1, ts: 1058000000000, entity-tag: entity1}
+ - {id: 904ea54c139fc8a4, ts: 1059000000000, entity-tag: entity1}
+ - {id: 32de75400f710ff8, ts: 1060000000000, entity-tag: entity1}
+ - {id: ce3d01f318f6b4dc, ts: 1061000000000, entity-tag: entity1}
+ - {id: 5495a87d0ee2ea26, ts: 1062000000000, entity-tag: entity1}
+ - {id: 51f32e7e1ac07d66, ts: 1063000000000, entity-tag: entity1}
+ - {id: 7af564e0f0692867, ts: 1064000000000, entity-tag: entity1}
+ - {id: 1a955f36ed7092d6, ts: 1065000000000, entity-tag: entity1}
+ - {id: 8a4e90bfb1b5ec40, ts: 1066000000000, entity-tag: entity1}
+ - {id: 38c2c0ef1915666e, ts: 1067000000000, entity-tag: entity1}
+ - {id: 2e116352a6973df3, ts: 1068000000000, entity-tag: entity1}
+ - {id: 9d7497ec79de4a9f, ts: 1069000000000, entity-tag: entity1}
+ - {id: 186b5a2305df9458, ts: 1070000000000, entity-tag: entity1}
+ - {id: 21550712d4634827, ts: 1071000000000, entity-tag: entity1}
+ - {id: 2445abfe0213211f, ts: 1072000000000, entity-tag: entity1}
+ - {id: 8f660868980400e9, ts: 1073000000000, entity-tag: entity1}
+ - {id: cb218b8208b0e7fc, ts: 1074000000000, entity-tag: entity1}
+ - {id: 5cbcc867a1c1cc4e, ts: 1075000000000, entity-tag: entity1}
+ - {id: 176cc8270665b4b1, ts: 1076000000000, entity-tag: entity1}
+ - {id: 6fd8bca0e66dc1f8, ts: 1077000000000, entity-tag: entity1}
+ - {id: 191f47536e2aff77, ts: 1078000000000, entity-tag: entity1}
+ - {id: 7569231c1ac4c7b6, ts: 1079000000000, entity-tag: entity1}
+ - {id: 2167701ce2b6485a, ts: 1080000000000, entity-tag: entity1}
+ - {id: ef0189f01d74e13a, ts: 1081000000000, entity-tag: entity1}
+ - {id: 669c3ae789d70b05, ts: 1082000000000, entity-tag: entity1}
+ - {id: c127b3e1de704175, ts: 1083000000000, entity-tag: entity1}
+ - {id: 05490894c797dc05, ts: 1084000000000, entity-tag: entity1}
+ - {id: 70301eae10180ffb, ts: 1085000000000, entity-tag: entity1}
+ - {id: 538f76130ce78677, ts: 1086000000000, entity-tag: entity1}
+ - {id: 2264c930c1826ede, ts: 1087000000000, entity-tag: entity1}
+ - {id: dfe4878677657c86, ts: 1088000000000, entity-tag: entity1}
+ - {id: e583430b847dde73, ts: 1089000000000, entity-tag: entity1}
+ - {id: 94ec7fdd317043c8, ts: 1090000000000, entity-tag: entity1}
+ - {id: e7843f11c5cd690c, ts: 1091000000000, entity-tag: entity1}
+ - {id: 527fe3c72304d4f2, ts: 1092000000000, entity-tag: entity1}
+ - {id: 72715e5596e66059, ts: 1093000000000, entity-tag: entity1}
+ - {id: 61a422be7acab6cf, ts: 1094000000000, entity-tag: entity1}
+ - {id: a76577849e24cac1, ts: 1095000000000, entity-tag: entity1}
+ - {id: d355a7a9992defa8, ts: 1096000000000, entity-tag: entity1}
+ - {id: 235a266aad1db83d, ts: 1097000000000, entity-tag: entity1}
+ - {id: d05f955d94401763, ts: 1098000000000, entity-tag: entity1}
+ - {id: f6ad4a0c7a9696b2, ts: 1099000000000, entity-tag: entity1}
+ - {id: c98ce4f234fc93dd, ts: 1100000000000, entity-tag: entity1}
+ - {id: 3886e225d9167124, ts: 1101000000000, entity-tag: entity1}
+ - {id: 7110c29fd8cd294e, ts: 1102000000000, entity-tag: entity1}
+ - {id: c97c8b05506361d4, ts: 1103000000000, entity-tag: entity1}
+ - {id: 6eda57b42c2ce7ea, ts: 1104000000000, entity-tag: entity1}
+ - {id: c09eca13d6ace55a, ts: 1105000000000, entity-tag: entity1}
+ - {id: 429a145794717000, ts: 1106000000000, entity-tag: entity1}
+ - {id: 70625aee23cfacc0, ts: 1107000000000, entity-tag: entity1}
+ - {id: 5b695d8264758b60, ts: 1108000000000, entity-tag: entity1}
+ - {id: 19df8b98a8bd73ad, ts: 1109000000000, entity-tag: entity1}
+ - {id: 3baf032d46de01d6, ts: 1110000000000, entity-tag: entity1}
+ - {id: 3551117be47f1ce9, ts: 1111000000000, entity-tag: entity1}
+ - {id: 0014024e85169465, ts: 1112000000000, entity-tag: entity1}
+ - {id: 1c63f1033c228ec9, ts: 1113000000000, entity-tag: entity1}
+ - {id: f6aa921be0d37b4f, ts: 1114000000000, entity-tag: entity1}
+ - {id: bed3dedeb62a2c65, ts: 1115000000000, entity-tag: entity1}
+ - {id: 3066332e1dda59bd, ts: 1116000000000, entity-tag: entity1}
+ - {id: 67ffcb944be48318, ts: 1117000000000, entity-tag: entity1}
+ - {id: 07bd1d928422e5da, ts: 1118000000000, entity-tag: entity1}
+ - {id: 61ed48256156df56, ts: 1119000000000, entity-tag: entity1}
+ - {id: d9a2fd5df755b9bb, ts: 1120000000000, entity-tag: entity1}
+ - {id: 1230b8283026b054, ts: 1121000000000, entity-tag: entity1}
+ - {id: a2f1075241fdbcdb, ts: 1122000000000, entity-tag: entity1}
+ - {id: 14b79999d640f496, ts: 1123000000000, entity-tag: entity1}
+ - {id: a470f77fa040a033, ts: 1124000000000, entity-tag: entity1}
+ - {id: 399cc5f5a6ef55c6, ts: 1125000000000, entity-tag: entity1}
+ - {id: af3bcdc64b488988, ts: 1126000000000, entity-tag: entity1}
+ - {id: fd5522fbbdbacb9b, ts: 1127000000000, entity-tag: entity1}
+ - {id: 55dad40440fb3d6a, ts: 1128000000000, entity-tag: entity1}
+ - {id: a12bcf56c0fafb6c, ts: 1129000000000, entity-tag: entity1}
+ - {id: c7c571a7186f9e8f, ts: 1130000000000, entity-tag: entity1}
+ - {id: dc588194b00fe810, ts: 1131000000000, entity-tag: entity1}
+ - {id: 91450864757d5e96, ts: 1132000000000, entity-tag: entity1}
+ - {id: 39c0b71299459383, ts: 1133000000000, entity-tag: entity1}
+ - {id: 5a1d5f37daf77a73, ts: 1134000000000, entity-tag: entity1}
+ - {id: c0f00aa0707b301d, ts: 1135000000000, entity-tag: entity1}
+ - {id: ad6e241c9c8454ac, ts: 1136000000000, entity-tag: entity1}
+ - {id: ee2ee555d52acd46, ts: 1137000000000, entity-tag: entity1}
+ - {id: 40be39c7627be6b5, ts: 1138000000000, entity-tag: entity1}
+ - {id: ce8a3a543409436c, ts: 1139000000000, entity-tag: entity1}
+ - {id: 5840b32f6a9b47e0, ts: 1140000000000, entity-tag: entity1}
+ - {id: ddb45dbb34160a4a, ts: 1141000000000, entity-tag: entity1}
+ - {id: 50ff58042e9d31c9, ts: 1142000000000, entity-tag: entity1}
+ - {id: a5a2ee86100c0330, ts: 1143000000000, entity-tag: entity1}
+ - {id: ddab9d9a09c13dc3, ts: 1144000000000, entity-tag: entity1}
+ - {id: 1650dda4c16cd75e, ts: 1145000000000, entity-tag: entity1}
+ - {id: 05da7fb3d1fa8799, ts: 1146000000000, entity-tag: entity1}
+ - {id: ec24981f79097d60, ts: 1147000000000, entity-tag: entity1}
+ - {id: ed83707ab7dc251a, ts: 1148000000000, entity-tag: entity1}
+ - {id: faacb4dd5c1d8801, ts: 1149000000000, entity-tag: entity1}
+ - {id: 13b6854d6ca1e7ca, ts: 1150000000000, entity-tag: entity1}
+ - {id: 9a0f07fe741ba93e, ts: 1151000000000, entity-tag: entity1}
+ - {id: d159eb914ed3eadc, ts: 1152000000000, entity-tag: entity1}
+ - {id: 74a27627e408d4b4, ts: 1153000000000, entity-tag: entity1}
+ - {id: 2057fc8c1e34952d, ts: 1154000000000, entity-tag: entity1}
+ - {id: d1f3adaa9e8d3358, ts: 1155000000000, entity-tag: entity1}
+ - {id: e7546db2819a8708, ts: 1156000000000, entity-tag: entity1}
+ - {id: 426c9b97c89c0451, ts: 1157000000000, entity-tag: entity1}
+ - {id: 00d7fcbddfd90275, ts: 1158000000000, entity-tag: entity1}
+ - {id: 88ef04e43aa4ec7f, ts: 1159000000000, entity-tag: entity1}
+ - {id: feaab95fbd168ff2, ts: 1160000000000, entity-tag: entity1}
+ - {id: ff1a20fc54ab40a9, ts: 1161000000000, entity-tag: entity1}
+ - {id: 4d776b743f381434, ts: 1162000000000, entity-tag: entity1}
+ - {id: d8d039bc77e0755e, ts: 1163000000000, entity-tag: entity1}
+ - {id: f6fa399a7c889a72, ts: 1164000000000, entity-tag: entity1}
+ - {id: 010a82b7cdd21124, ts: 1165000000000, entity-tag: entity1}
+ - {id: ccf6996272da9ec1, ts: 1166000000000, entity-tag: entity1}
+ - {id: c8350cd244070f6b, ts: 1167000000000, entity-tag: entity1}
+ - {id: ed507e9a7d578205, ts: 1168000000000, entity-tag: entity1}
+ - {id: e1b2bf9964e28f66, ts: 1169000000000, entity-tag: entity1}
+ - {id: 5618c6b2e9745cf6, ts: 1170000000000, entity-tag: entity1}
+ - {id: a0fd444d821813bd, ts: 1171000000000, entity-tag: entity1}
+ - {id: af54f8b9f8db3137, ts: 1172000000000, entity-tag: entity1}
+ - {id: 49579d1c7b3dd999, ts: 1173000000000, entity-tag: entity1}
+ - {id: 5e8d64e778d3bf5f, ts: 1174000000000, entity-tag: entity1}
+ - {id: de47eb7cc57ec9a7, ts: 1175000000000, entity-tag: entity1}
+ - {id: a2d726aab97de968, ts: 1176000000000, entity-tag: entity1}
+ - {id: 3cadeb8f83842b3d, ts: 1177000000000, entity-tag: entity1}
+ - {id: 83773faee2004045, ts: 1178000000000, entity-tag: entity1}
+ - {id: 765172e0c34408f4, ts: 1179000000000, entity-tag: entity1}
+ - {id: 75f5e52f3053f3a4, ts: 1180000000000, entity-tag: entity1}
+ - {id: 37f0d6a72c9d502e, ts: 1181000000000, entity-tag: entity1}
+ - {id: 6b42c8d70d2ebbdc, ts: 1182000000000, entity-tag: entity1}
+ - {id: cd95988846dad489, ts: 1183000000000, entity-tag: entity1}
+ - {id: 0c7731fb5b187107, ts: 1184000000000, entity-tag: entity1}
+ - {id: f3418985f0ddc857, ts: 1185000000000, entity-tag: entity1}
+ - {id: 85eeb5c953098f09, ts: 1186000000000, entity-tag: entity1}
+ - {id: 818a89bdf7108634, ts: 1187000000000, entity-tag: entity1}
+ - {id: 9f226b08a82d0aec, ts: 1188000000000, entity-tag: entity1}
+ - {id: 999d4e1762046b1e, ts: 1189000000000, entity-tag: entity1}
+ - {id: 690b780d3f7cae88, ts: 1190000000000, entity-tag: entity1}
+ - {id: 539f4ab03d17052b, ts: 1191000000000, entity-tag: entity1}
+ - {id: 259b72017ed1b27e, ts: 1192000000000, entity-tag: entity1}
+ - {id: 28e5556cf5dbe691, ts: 1193000000000, entity-tag: entity1}
+ - {id: 2cbdd5846a3eb256, ts: 1194000000000, entity-tag: entity1}
+ - {id: 86a62f6280717e33, ts: 1195000000000, entity-tag: entity1}
+ - {id: 97d2efa0637f57f3, ts: 1196000000000, entity-tag: entity1}
+ - {id: 5aa5f5c5e8b29fc6, ts: 1197000000000, entity-tag: entity1}
+ - {id: e3afb5136c21577d, ts: 1198000000000, entity-tag: entity1}
+ - {id: 79a4b6f753be3853, ts: 1199000000000, entity-tag: entity1}
+ - {id: bc07d1040c034c4e, ts: 1200000000000, entity-tag: entity1}
+ - {id: 81d1372f5c587a3f, ts: 1201000000000, entity-tag: entity1}
+ - {id: 0b6523877ab5957e, ts: 1202000000000, entity-tag: entity1}
+ - {id: 6090aec10765b0ad, ts: 1203000000000, entity-tag: entity1}
+ - {id: ad8303c41f25d8b4, ts: 1204000000000, entity-tag: entity1}
+ - {id: fc09b6a1df1fb505, ts: 1205000000000, entity-tag: entity1}
+ - {id: 673246ead0d4f10e, ts: 1206000000000, entity-tag: entity1}
+ - {id: a26de7e6bbd00dfb, ts: 1207000000000, entity-tag: entity1}
+ - {id: 5559d736e0cf1447, ts: 1208000000000, entity-tag: entity1}
+ - {id: 1dafc370294d8fdb, ts: 1209000000000, entity-tag: entity1}
+ - {id: d1d3d339d191caca, ts: 1210000000000, entity-tag: entity1}
+ - {id: a9ca8db555d515a2, ts: 1211000000000, entity-tag: entity1}
+ - {id: 67a9f53da0e48421, ts: 1212000000000, entity-tag: entity1}
+ - {id: 54ba9f7191740dd6, ts: 1213000000000, entity-tag: entity1}
+ - {id: 4f5e4c3be7623f7b, ts: 1214000000000, entity-tag: entity1}
+ - {id: 6bcd1996db325de8, ts: 1215000000000, entity-tag: entity1}
+ - {id: abdc3702428e590f, ts: 1216000000000, entity-tag: entity1}
+ - {id: 3368dc224f455fd6, ts: 1217000000000, entity-tag: entity1}
+ - {id: fb07e00e0cfedc3b, ts: 1218000000000, entity-tag: entity1}
+ - {id: de2c0a42b8d82408, ts: 1219000000000, entity-tag: entity1}
+ - {id: 82078346a66c4185, ts: 1220000000000, entity-tag: entity1}
+ - {id: 00e4e7dc286ea171, ts: 1221000000000, entity-tag: entity1}
+ - {id: b08833d2c03e11c0, ts: 1222000000000, entity-tag: entity1}
+ - {id: 84688f2920d25c1b, ts: 1223000000000, entity-tag: entity1}
+ - {id: 1ea9c7362006237e, ts: 1224000000000, entity-tag: entity1}
+ - {id: db78dfcd7ebbeac6, ts: 1225000000000, entity-tag: entity1}
+ - {id: cd8816089cfeb2f5, ts: 1226000000000, entity-tag: entity1}
+ - {id: 3bf8803c01a3438a, ts: 1227000000000, entity-tag: entity1}
+ - {id: d99c0c43bdffa6c0, ts: 1228000000000, entity-tag: entity1}
+ - {id: 9a94cb220653a2af, ts: 1229000000000, entity-tag: entity1}
+ - {id: a747f3735efeff3b, ts: 1230000000000, entity-tag: entity1}
+ - {id: 213f0b262db9319f, ts: 1231000000000, entity-tag: entity1}
+ - {id: 72fab30e88dc3b64, ts: 1232000000000, entity-tag: entity1}
+ - {id: 5e63795ece28908f, ts: 1233000000000, entity-tag: entity1}
+ - {id: 67a7b9b437fc3321, ts: 1234000000000, entity-tag: entity1}
+ - {id: 21c77be3461bd651, ts: 1235000000000, entity-tag: entity1}
+ - {id: 8bc62e0414cc6182, ts: 1236000000000, entity-tag: entity1}
+ - {id: ba1dbe87bd8547c4, ts: 1237000000000, entity-tag: entity1}
+ - {id: 9552186202ce0eea, ts: 1238000000000, entity-tag: entity1}
+ - {id: d923c88d83f756fb, ts: 1239000000000, entity-tag: entity1}
+ - {id: 12c43336a63b317d, ts: 1240000000000, entity-tag: entity1}
+ - {id: 44af84de4ba1b4d7, ts: 1241000000000, entity-tag: entity1}
+ - {id: ad301c51a0bd04ca, ts: 1242000000000, entity-tag: entity1}
+ - {id: b433d171fa7d1bd3, ts: 1243000000000, entity-tag: entity1}
+ - {id: cb54fc0cab5ba8a9, ts: 1244000000000, entity-tag: entity1}
+ - {id: 5ee2dea36fb18cb8, ts: 1245000000000, entity-tag: entity1}
+ - {id: 7e7ce6c98c2ac382, ts: 1246000000000, entity-tag: entity1}
+ - {id: 7310cdd1e122db0d, ts: 1247000000000, entity-tag: entity1}
+ - {id: bd39862aac20d0c6, ts: 1248000000000, entity-tag: entity1}
+ - {id: 2bd9497ce7b3d7f4, ts: 1249000000000, entity-tag: entity1}
+ - {id: 984c37cba88818c5, ts: 1250000000000, entity-tag: entity1}
+ - {id: 65797871e533ae7e, ts: 1251000000000, entity-tag: entity1}
+ - {id: f481adc8fbd2c78e, ts: 1252000000000, entity-tag: entity1}
+ - {id: db2ca3c72da54de7, ts: 1253000000000, entity-tag: entity1}
+ - {id: 8d4a25e9a7871f1c, ts: 1254000000000, entity-tag: entity1}
+ - {id: 4a334997034e9ba9, ts: 1255000000000, entity-tag: entity1}
+ - {id: 52650b94ac30c905, ts: 1256000000000, entity-tag: entity1}
+ - {id: c8679e24f8b200dc, ts: 1257000000000, entity-tag: entity1}
+ - {id: 5828f44c297b7a77, ts: 1258000000000, entity-tag: entity1}
+ - {id: ad93b4a0df2ba995, ts: 1259000000000, entity-tag: entity1}
+ - {id: d0536f3d4fccf195, ts: 1260000000000, entity-tag: entity1}
+ - {id: 347cc4b85991c4b8, ts: 1261000000000, entity-tag: entity1}
+ - {id: 02dca8a1840acd83, ts: 1262000000000, entity-tag: entity1}
+ - {id: 47909f72c09d85e7, ts: 1263000000000, entity-tag: entity1}
+ - {id: b37a0ccc266eb360, ts: 1264000000000, entity-tag: entity1}
+ - {id: 7a56f929bcff759d, ts: 1265000000000, entity-tag: entity1}
+ - {id: 1588a73fe13784b8, ts: 1266000000000, entity-tag: entity1}
+ - {id: af084a2c8e48b8af, ts: 1267000000000, entity-tag: entity1}
+ - {id: 0492a26ab80496f3, ts: 1268000000000, entity-tag: entity1}
+ - {id: f00fa157ebbad72a, ts: 1269000000000, entity-tag: entity1}
+ - {id: 2f2c85a7b3d07ca7, ts: 1270000000000, entity-tag: entity1}
+ - {id: 630cc907bf046369, ts: 1271000000000, entity-tag: entity1}
+ - {id: b9f5182c6fac570a, ts: 1272000000000, entity-tag: entity1}
+ - {id: a847192273edaa26, ts: 1273000000000, entity-tag: entity1}
+ - {id: 60c8f90cb9ee7db5, ts: 1274000000000, entity-tag: entity1}
+ - {id: 585a81816f12cd22, ts: 1275000000000, entity-tag: entity1}
+ - {id: c9ae2ecd3ec89b6d, ts: 1276000000000, entity-tag: entity1}
+ - {id: 02aee7e5f310d6f1, ts: 1277000000000, entity-tag: entity1}
+ - {id: 23e53023209d6f64, ts: 1278000000000, entity-tag: entity1}
+ - {id: e01a483eb7be380b, ts: 1279000000000, entity-tag: entity1}
+ - {id: 9e839fc2b6b47b13, ts: 1280000000000, entity-tag: entity1}
+ - {id: ed4b5794a324e52a, ts: 1281000000000, entity-tag: entity1}
+ - {id: a10639604261ec2a, ts: 1282000000000, entity-tag: entity1}
+ - {id: dc191cfc85123ba5, ts: 1283000000000, entity-tag: entity1}
+ - {id: 1a4dc05e7c49e0b0, ts: 1284000000000, entity-tag: entity1}
+ - {id: f57ddb765ffd918c, ts: 1285000000000, entity-tag: entity1}
+ - {id: 51fd2f538d26d0d0, ts: 1286000000000, entity-tag: entity1}
+ - {id: 09634e5b3c2c1663, ts: 1287000000000, entity-tag: entity1}
+ - {id: 65724362f92458bc, ts: 1288000000000, entity-tag: entity1}
+ - {id: c56525ee5da420f9, ts: 1289000000000, entity-tag: entity1}
+ - {id: 5d90705658f027b6, ts: 1290000000000, entity-tag: entity1}
+ - {id: ed68e1720b4eb945, ts: 1291000000000, entity-tag: entity1}
+ - {id: 68eec83ebb9845f6, ts: 1292000000000, entity-tag: entity1}
+ - {id: 69ea23f5be54525e, ts: 1293000000000, entity-tag: entity1}
+ - {id: fcc887f2cd7f8bc9, ts: 1294000000000, entity-tag: entity1}
+ - {id: d72cddb19ed66e6e, ts: 1295000000000, entity-tag: entity1}
+ - {id: d0094f420b53c0c5, ts: 1296000000000, entity-tag: entity1}
+ - {id: bdd824e149e3a2c4, ts: 1297000000000, entity-tag: entity1}
+ - {id: 636f4e10e7f55845, ts: 1298000000000, entity-tag: entity1}
+ - {id: 474fe1ba63723a54, ts: 1299000000000, entity-tag: entity1}
+ - {id: f27c2a5e41c29ce2, ts: 1300000000000, entity-tag: entity1}
+ - {id: c594fcdf131f7d44, ts: 1301000000000, entity-tag: entity1}
+ - {id: 0306ece5d8ae0201, ts: 1302000000000, entity-tag: entity1}
+ - {id: 9d501eabfad5f79d, ts: 1303000000000, entity-tag: entity1}
+ - {id: 7c3c8024cfac2eb8, ts: 1304000000000, entity-tag: entity1}
+ - {id: ac4df28d2f3c8122, ts: 1305000000000, entity-tag: entity1}
+ - {id: 046808131b9a6653, ts: 1306000000000, entity-tag: entity1}
+ - {id: d0fd0e3ea3a57aec, ts: 1307000000000, entity-tag: entity1}
+ - {id: e110e713204fd340, ts: 1308000000000, entity-tag: entity1}
+ - {id: 814c03f8c2c9ae91, ts: 1309000000000, entity-tag: entity1}
+ - {id: 0710005597013ed1, ts: 1310000000000, entity-tag: entity1}
+ - {id: a646e7e8a0259f9b, ts: 1311000000000, entity-tag: entity1}
+ - {id: 3a29145ee1f4b4db, ts: 1312000000000, entity-tag: entity1}
+ - {id: 2ad1e6dd099fb500, ts: 1313000000000, entity-tag: entity1}
+ - {id: ddfb1c837861f01d, ts: 1314000000000, entity-tag: entity1}
+ - {id: b3c5086db4ccdde6, ts: 1315000000000, entity-tag: entity1}
+ - {id: d7731adf32af093d, ts: 1316000000000, entity-tag: entity1}
+ - {id: ef5091d772741fc7, ts: 1317000000000, entity-tag: entity1}
+ - {id: 827e21fdac6ab3f5, ts: 1318000000000, entity-tag: entity1}
+ - {id: fa11f52873a4bbb4, ts: 1319000000000, entity-tag: entity1}
+ - {id: 6bacadecc572950d, ts: 1320000000000, entity-tag: entity1}
+ - {id: d58d31566ec72e4a, ts: 1321000000000, entity-tag: entity1}
+ - {id: 59edeed2a5f0fefe, ts: 1322000000000, entity-tag: entity1}
+ - {id: f54e280292cab8a4, ts: 1323000000000, entity-tag: entity1}
+ - {id: 10607025c6b27e1e, ts: 1324000000000, entity-tag: entity1}
+ - {id: 0c87df7d682800e9, ts: 1325000000000, entity-tag: entity1}
+ - {id: a3b8be2ff53a34e4, ts: 1326000000000, entity-tag: entity1}
+ - {id: b87f66d420cf73a0, ts: 1327000000000, entity-tag: entity1}
+ - {id: fbfaa1fa805a7dab, ts: 1328000000000, entity-tag: entity1}
+ - {id: cb745d087e89fb09, ts: 1329000000000, entity-tag: entity1}
+ - {id: ea1748c97fe61fea, ts: 1330000000000, entity-tag: entity1}
+ - {id: 2c5afcebba86a01a, ts: 1331000000000, entity-tag: entity1}
+ - {id: 6a479fc0220b981d, ts: 1332000000000, entity-tag: entity1}
+ - {id: 5625b03411e3bcee, ts: 1333000000000, entity-tag: entity1}
+ - {id: fc90da3160aedc45, ts: 1334000000000, entity-tag: entity1}
+ - {id: 0db0b9cb2a9f530f, ts: 1335000000000, entity-tag: entity1}
+ - {id: 3dcbf71ce191bd4c, ts: 1336000000000, entity-tag: entity1}
+ - {id: a3877dcc139b284f, ts: 1337000000000, entity-tag: entity1}
+ - {id: 11f2a2ce87a096e2, ts: 1338000000000, entity-tag: entity1}
+ - {id: f1d4fddf3e381270, ts: 1339000000000, entity-tag: entity1}
+ - {id: 4dcebbe7511a7b12, ts: 1340000000000, entity-tag: entity1}
+ - {id: a9804360f30ec30f, ts: 1341000000000, entity-tag: entity1}
+ - {id: 40606a7bcee665e2, ts: 1342000000000, entity-tag: entity1}
+ - {id: d1060a9b6b616d4b, ts: 1343000000000, entity-tag: entity1}
+ - {id: 6df7314fff2f2045, ts: 1344000000000, entity-tag: entity1}
+ - {id: 4ce29480745f6b5e, ts: 1345000000000, entity-tag: entity1}
+ - {id: 0459e38d47e365f5, ts: 1346000000000, entity-tag: entity1}
+ - {id: 897fa45579a81abf, ts: 1347000000000, entity-tag: entity1}
+ - {id: d4e68beb19e9d8ef, ts: 1348000000000, entity-tag: entity1}
+ - {id: 5c4feb7f374c722e, ts: 1349000000000, entity-tag: entity1}
+ - {id: c26312888070c898, ts: 1350000000000, entity-tag: entity1}
+ - {id: 859141546c976b74, ts: 1351000000000, entity-tag: entity1}
+ - {id: a31be7ba2a8a4271, ts: 1352000000000, entity-tag: entity1}
+ - {id: 82d92d87d01ba69b, ts: 1353000000000, entity-tag: entity1}
+ - {id: a59389fdcfd95e62, ts: 1354000000000, entity-tag: entity1}
+ - {id: b1bd3fa0ae0f9dbf, ts: 1355000000000, entity-tag: entity1}
+ - {id: d5ed415c3e4ff266, ts: 1356000000000, entity-tag: entity1}
+ - {id: da8e8c729abee6ed, ts: 1357000000000, entity-tag: entity1}
+ - {id: 21ae67a5ed0fb805, ts: 1358000000000, entity-tag: entity1}
+ - {id: 6ad286f52541c8cc, ts: 1359000000000, entity-tag: entity1}
+ - {id: 16a114c658834609, ts: 1360000000000, entity-tag: entity1}
+ - {id: 452b349b40549aa9, ts: 1361000000000, entity-tag: entity1}
+ - {id: 3fe7b2b5bc3df525, ts: 1362000000000, entity-tag: entity1}
+ - {id: 2a20797a930f51a2, ts: 1363000000000, entity-tag: entity1}
+ - {id: deae3391d8486812, ts: 1364000000000, entity-tag: entity1}
+ - {id: bf06a81c564bec82, ts: 1365000000000, entity-tag: entity1}
+ - {id: 9ed4eb6916788a84, ts: 1366000000000, entity-tag: entity1}
+ - {id: 03cf42d5960052e9, ts: 1367000000000, entity-tag: entity1}
+ - {id: dea4ded92685c284, ts: 1368000000000, entity-tag: entity1}
+ - {id: 3c6a6bd87c1568ba, ts: 1369000000000, entity-tag: entity1}
+ - {id: a790578db8eb3b4e, ts: 1370000000000, entity-tag: entity1}
+ - {id: 61c9b0d75e65935d, ts: 1371000000000, entity-tag: entity1}
+ - {id: 11e8c70267bc2d18, ts: 1372000000000, entity-tag: entity1}
+ - {id: ba487ec8cdae000b, ts: 1373000000000, entity-tag: entity1}
+ - {id: c18292aedf7173cb, ts: 1374000000000, entity-tag: entity1}
+ - {id: 5c8c38e74a347e35, ts: 1375000000000, entity-tag: entity1}
+ - {id: 7c3d362c61f66337, ts: 1376000000000, entity-tag: entity1}
+ - {id: 05c5fa39cc14ba6e, ts: 1377000000000, entity-tag: entity1}
+ - {id: 4b0ae0a3f877ce90, ts: 1378000000000, entity-tag: entity1}
+ - {id: a3b4b8588586b5cb, ts: 1379000000000, entity-tag: entity1}
+ - {id: f3fec947ea6c62bb, ts: 1380000000000, entity-tag: entity1}
+ - {id: 2f5297d7f5b7def7, ts: 1381000000000, entity-tag: entity1}
+ - {id: 788952e908623690, ts: 1382000000000, entity-tag: entity1}
+ - {id: aa7e71912bfdf561, ts: 1383000000000, entity-tag: entity1}
+ - {id: 67d576ec83eae3f0, ts: 1384000000000, entity-tag: entity1}
+ - {id: 2d6e74d269157630, ts: 1385000000000, entity-tag: entity1}
+ - {id: 642d87de72f14887, ts: 1386000000000, entity-tag: entity1}
+ - {id: 79d78ecf2a4f8ab7, ts: 1387000000000, entity-tag: entity1}
+ - {id: 49ac45161526ff1b, ts: 1388000000000, entity-tag: entity1}
+ - {id: 79ce29da7b2b3554, ts: 1389000000000, entity-tag: entity1}
+ - {id: 8a60e7afb980c988, ts: 1390000000000, entity-tag: entity1}
+ - {id: e7249a0d0d1f9eb0, ts: 1391000000000, entity-tag: entity1}
+ - {id: bf9154886db19052, ts: 1392000000000, entity-tag: entity1}
+ - {id: 4364d2d41ac8c04b, ts: 1393000000000, entity-tag: entity1}
+ - {id: e6b1cf9de5d5ae3c, ts: 1394000000000, entity-tag: entity1}
+ - {id: b14243de1ae92192, ts: 1395000000000, entity-tag: entity1}
+ - {id: 7d73012a96ebe74e, ts: 1396000000000, entity-tag: entity1}
+ - {id: c5d30a6584e10e11, ts: 1397000000000, entity-tag: entity1}
+ - {id: 6c1801908534dd7e, ts: 1398000000000, entity-tag: entity1}
+ - {id: d42163fa1379257e, ts: 1399000000000, entity-tag: entity1}
+ - {id: 324f0b11f37a0b42, ts: 1400000000000, entity-tag: entity1}
+ - {id: d28bda585a592816, ts: 1401000000000, entity-tag: entity1}
+ - {id: 97eff389c89bcd89, ts: 1402000000000, entity-tag: entity1}
+ - {id: e55d797a20b8196e, ts: 1403000000000, entity-tag: entity1}
+ - {id: b4bf3897a3a4cb23, ts: 1404000000000, entity-tag: entity1}
+ - {id: ef642728add20064, ts: 1405000000000, entity-tag: entity1}
+ - {id: 44d4c978e021dd01, ts: 1406000000000, entity-tag: entity1}
+ - {id: a906e039125e8242, ts: 1407000000000, entity-tag: entity1}
+ - {id: 9493cd7400fded1d, ts: 1408000000000, entity-tag: entity1}
+ - {id: 6475b25bef37964c, ts: 1409000000000, entity-tag: entity1}
+ - {id: 5ade0c1d5f45d2da, ts: 1410000000000, entity-tag: entity1}
+ - {id: d0b48de31d979e82, ts: 1411000000000, entity-tag: entity1}
+ - {id: a6f212222f28c1ba, ts: 1412000000000, entity-tag: entity1}
+ - {id: ed6a4ba6f1e11a9e, ts: 1413000000000, entity-tag: entity1}
+ - {id: b1aa7fb49d03368e, ts: 1414000000000, entity-tag: entity1}
+ - {id: a80e53d25d080301, ts: 1415000000000, entity-tag: entity1}
+ - {id: d08ec943fe931c8b, ts: 1416000000000, entity-tag: entity1}
+ - {id: 780bd5272c20b35f, ts: 1417000000000, entity-tag: entity1}
+ - {id: 33be7082a2a92563, ts: 1418000000000, entity-tag: entity1}
+ - {id: db82e92134c4d5cd, ts: 1419000000000, entity-tag: entity1}
+ - {id: 843fb04202197a9c, ts: 1420000000000, entity-tag: entity1}
+ - {id: 5bb0345672fe7f15, ts: 1421000000000, entity-tag: entity1}
+ - {id: d3c8639a9a42e888, ts: 1422000000000, entity-tag: entity1}
+ - {id: 08450ed6368f16a5, ts: 1423000000000, entity-tag: entity1}
+ - {id: 9ab9b718c3c377d0, ts: 1424000000000, entity-tag: entity1}
+ - {id: 3b4444245ef2046a, ts: 1425000000000, entity-tag: entity1}
+ - {id: 86285ce3a799f544, ts: 1426000000000, entity-tag: entity1}
+ - {id: 8229d4e4e1a550c2, ts: 1427000000000, entity-tag: entity1}
+ - {id: f519d82ec30cd99d, ts: 1428000000000, entity-tag: entity1}
+ - {id: 3c2f257c7accbc01, ts: 1429000000000, entity-tag: entity1}
+ - {id: fe37f8b4fa33541c, ts: 1430000000000, entity-tag: entity1}
+ - {id: 6679ab1ddc48f902, ts: 1431000000000, entity-tag: entity1}
+ - {id: 4b64b3f7075dd5b6, ts: 1432000000000, entity-tag: entity1}
+ - {id: b562bf6ca29076cc, ts: 1433000000000, entity-tag: entity1}
+ - {id: c6f93a329e1c5ef8, ts: 1434000000000, entity-tag: entity1}
+ - {id: 519c43e7f323aa78, ts: 1435000000000, entity-tag: entity1}
+ - {id: 1b855dc84d78ea01, ts: 1436000000000, entity-tag: entity1}
+ - {id: f8c5d0a6badda485, ts: 1437000000000, entity-tag: entity1}
+ - {id: 114f7de9da2d5882, ts: 1438000000000, entity-tag: entity1}
+ - {id: 246c70f3d54dacc6, ts: 1439000000000, entity-tag: entity1}
+ - {id: 874aa87c490142a6, ts: 1440000000000, entity-tag: entity1}
+ - {id: aaa187e8823cdbac, ts: 1441000000000, entity-tag: entity1}
+ - {id: f66f01247525eba1, ts: 1442000000000, entity-tag: entity1}
+ - {id: 39a424c78892c94e, ts: 1443000000000, entity-tag: entity1}
+ - {id: 064b5a0b1900e349, ts: 1444000000000, entity-tag: entity1}
+ - {id: 31884156a4376baf, ts: 1445000000000, entity-tag: entity1}
+ - {id: b8baca580932560a, ts: 1446000000000, entity-tag: entity1}
+ - {id: b92330bd78fb5147, ts: 1447000000000, entity-tag: entity1}
+ - {id: 9923a3b769dcc090, ts: 1448000000000, entity-tag: entity1}
+ - {id: 6aa787924f09d2cf, ts: 1449000000000, entity-tag: entity1}
+ - {id: 808dc2aa3991d0f5, ts: 1450000000000, entity-tag: entity1}
+ - {id: 1599d458d7fb43ab, ts: 1451000000000, entity-tag: entity1}
+ - {id: 4a91d9913078d346, ts: 1452000000000, entity-tag: entity1}
+ - {id: d574ca71c61d4120, ts: 1453000000000, entity-tag: entity1}
+ - {id: 6cfbbaff9fe9f509, ts: 1454000000000, entity-tag: entity1}
+ - {id: adf529605e629e67, ts: 1455000000000, entity-tag: entity1}
+ - {id: 56c1486657dc30da, ts: 1456000000000, entity-tag: entity1}
+ - {id: 270adbe34e71deff, ts: 1457000000000, entity-tag: entity1}
+ - {id: 8d30cfade670c727, ts: 1458000000000, entity-tag: entity1}
+ - {id: 16d970023cd555fb, ts: 1459000000000, entity-tag: entity1}
+ - {id: 0aae31c17b198f89, ts: 1460000000000, entity-tag: entity1}
+ - {id: 729805251a4fd16e, ts: 1461000000000, entity-tag: entity1}
+ - {id: e9d20d8c6cb8a932, ts: 1462000000000, entity-tag: entity1}
+ - {id: 595b7ddb33a8080b, ts: 1463000000000, entity-tag: entity1}
+ - {id: 5890d3ec70d55e5d, ts: 1464000000000, entity-tag: entity1}
+ - {id: 2ac56f12d28aa538, ts: 1465000000000, entity-tag: entity1}
+ - {id: fba0ce6f3459110a, ts: 1466000000000, entity-tag: entity1}
+ - {id: 6b91382f5e638978, ts: 1467000000000, entity-tag: entity1}
+ - {id: 44948c6fa26ddbb2, ts: 1468000000000, entity-tag: entity1}
+ - {id: b5e6804bf4f88521, ts: 1469000000000, entity-tag: entity1}
+ - {id: 6ab427c4e67030c3, ts: 1470000000000, entity-tag: entity1}
+ - {id: 493ebe8fd6afdd79, ts: 1471000000000, entity-tag: entity1}
+ - {id: 95bebcb75bed871b, ts: 1472000000000, entity-tag: entity1}
+ - {id: 1a5a0a809d49ccaa, ts: 1473000000000, entity-tag: entity1}
+ - {id: 29a2546ab8d9ba24, ts: 1474000000000, entity-tag: entity1}
+ - {id: b5a67bb99a8364dd, ts: 1475000000000, entity-tag: entity1}
+ - {id: bd7a04722c8dfda2, ts: 1476000000000, entity-tag: entity1}
+ - {id: 3f639762c47b1a1d, ts: 1477000000000, entity-tag: entity1}
+ - {id: 8c4b9eb64ea2ba78, ts: 1478000000000, entity-tag: entity1}
+ - {id: 01943b98ede143b7, ts: 1479000000000, entity-tag: entity1}
+ - {id: e445fd6193047fe7, ts: 1480000000000, entity-tag: entity1}
+ - {id: ece95183d12ec319, ts: 1481000000000, entity-tag: entity1}
+ - {id: db3ba646842e57e8, ts: 1482000000000, entity-tag: entity1}
+ - {id: b4348df63e790bdc, ts: 1483000000000, entity-tag: entity1}
+ - {id: 1cb8d2e56f75807d, ts: 1484000000000, entity-tag: entity1}
+ - {id: 67475634e92e8091, ts: 1485000000000, entity-tag: entity1}
+ - {id: 1c4e1d5d39ce803d, ts: 1486000000000, entity-tag: entity1}
+ - {id: 9cbb70c5ef0df99d, ts: 1487000000000, entity-tag: entity1}
+ - {id: 72d52b1e11fe3fc6, ts: 1488000000000, entity-tag: entity1}
+ - {id: 8ea44d4256ca9701, ts: 1489000000000, entity-tag: entity1}
+ - {id: a7ef72931e9e570e, ts: 1490000000000, entity-tag: entity1}
+ - {id: c244815fab5e7075, ts: 1491000000000, entity-tag: entity1}
+ - {id: abff0689814b2cd5, ts: 1492000000000, entity-tag: entity1}
+ - {id: a00a334c5b640a05, ts: 1493000000000, entity-tag: entity1}
+ - {id: 4f7cbc2eeb85050c, ts: 1494000000000, entity-tag: entity1}
+ - {id: df855d6246b79726, ts: 1495000000000, entity-tag: entity1}
+ - {id: 933272c63979348c, ts: 1496000000000, entity-tag: entity1}
+ - {id: 7eeb2c08cf649360, ts: 1497000000000, entity-tag: entity1}
+ - {id: a5c05e8cd7ccb3b2, ts: 1498000000000, entity-tag: entity1}
+ - {id: a76e1dbef95d55d1, ts: 1499000000000, entity-tag: entity1}
diff --git a/banyand/stream/testdata/parity_topn_desc.yaml b/banyand/stream/testdata/parity_topn_desc.yaml
new file mode 100644
index 000000000..e339a5d21
--- /dev/null
+++ b/banyand/stream/testdata/parity_topn_desc.yaml
@@ -0,0 +1,27 @@
+# Golden expectation captured from the ROW query path (stream.Query ->
+# BuildElementsFromStreamResult) on branch rowdrop/base, before the row path was
+# removed (apache/skywalking#13998). It cannot be re-captured afterwards.
+#
+# Normalization, forced by the fixture generator:
+# ts - nanosecond offset from the query time range start. generateData anchors
+# element timestamps at time.Now(), so absolute timestamps differ per run.
+# tags - only entity-tag is recorded. The sibling filter-tag is drawn from
+# crypto/rand per run (benchmark_test.go generateRandomNumber), so its
+# value is not reproducible and no golden can pin it.
+# Element order is as the row path returned it.
+name: parity_topn_desc
+cases:
+ - name: desc
+ query: fixture{ts=100,series=1} order=ts-desc max=10
+ element_count: 10
+ elements:
+ - {id: 653cbe3860b2f8b8, ts: 99000000000, entity-tag: entity1}
+ - {id: bae7b044cf1f3238, ts: 98000000000, entity-tag: entity1}
+ - {id: e6c327a35bc00991, ts: 97000000000, entity-tag: entity1}
+ - {id: 25faf71b4e2c9b9b, ts: 96000000000, entity-tag: entity1}
+ - {id: d2688b2e323a5e6c, ts: 95000000000, entity-tag: entity1}
+ - {id: cfb0945e07a4b99e, ts: 94000000000, entity-tag: entity1}
+ - {id: 2897044e46e6b512, ts: 93000000000, entity-tag: entity1}
+ - {id: ab77a7816f6952ba, ts: 92000000000, entity-tag: entity1}
+ - {id: 79f4994c063b03fe, ts: 91000000000, entity-tag: entity1}
+ - {id: e4346756ceb39846, ts: 90000000000, entity-tag: entity1}
diff --git a/banyand/stream/testdata/parity_topn_multiseries_asc.yaml b/banyand/stream/testdata/parity_topn_multiseries_asc.yaml
new file mode 100644
index 000000000..946bfa883
--- /dev/null
+++ b/banyand/stream/testdata/parity_topn_multiseries_asc.yaml
@@ -0,0 +1,47 @@
+# Golden expectation captured from the ROW query path (stream.Query ->
+# BuildElementsFromStreamResult) on branch rowdrop/base, before the row path was
+# removed (apache/skywalking#13998). It cannot be re-captured afterwards.
+#
+# Normalization, forced by the fixture generator:
+# ts - nanosecond offset from the query time range start. generateData anchors
+# element timestamps at time.Now(), so absolute timestamps differ per run.
+# tags - only entity-tag is recorded. The sibling filter-tag is drawn from
+# crypto/rand per run (benchmark_test.go generateRandomNumber), so its
+# value is not reproducible and no golden can pin it.
+# Element order is as the row path returned it.
+name: parity_topn_multiseries_asc
+cases:
+ - name: asc
+ query: fixture{ts=40,series=5} order=ts-asc max=30
+ element_count: 30
+ elements:
+ - {id: 228d3fa5008c9ec2, ts: 0, entity-tag: entity1}
+ - {id: a224fe72b38b9e57, ts: 0, entity-tag: entity3}
+ - {id: bc94b2dc46775a9f, ts: 0, entity-tag: entity5}
+ - {id: 78c64d47e69118b9, ts: 0, entity-tag: entity4}
+ - {id: 537fef459947c66f, ts: 0, entity-tag: entity2}
+ - {id: 1a2698b8a9775f34, ts: 1000000000, entity-tag: entity2}
+ - {id: 3880e6e1dc9cb59a, ts: 1000000000, entity-tag: entity5}
+ - {id: 5460f49adbe7aba2, ts: 1000000000, entity-tag: entity1}
+ - {id: bb62ce8f2e1f2c09, ts: 1000000000, entity-tag: entity3}
+ - {id: 6de6f5d076d742b9, ts: 1000000000, entity-tag: entity4}
+ - {id: 6090b2aae0582978, ts: 2000000000, entity-tag: entity4}
+ - {id: 22e84c6060359bac, ts: 2000000000, entity-tag: entity1}
+ - {id: 511172adf1230e52, ts: 2000000000, entity-tag: entity2}
+ - {id: 5aa57a07d47ee7ea, ts: 2000000000, entity-tag: entity5}
+ - {id: 495e5792411ce777, ts: 2000000000, entity-tag: entity3}
+ - {id: 5e8d522ca93563ed, ts: 3000000000, entity-tag: entity3}
+ - {id: 19a82d369be44096, ts: 3000000000, entity-tag: entity2}
+ - {id: 7fee13f22a02b186, ts: 3000000000, entity-tag: entity4}
+ - {id: 9deb1683a072e62a, ts: 3000000000, entity-tag: entity1}
+ - {id: 6c7cab198703ee9f, ts: 3000000000, entity-tag: entity5}
+ - {id: bebfbdcd407e2ccf, ts: 4000000000, entity-tag: entity5}
+ - {id: fd6817dc570d1402, ts: 4000000000, entity-tag: entity4}
+ - {id: 7410f56e4c878bd6, ts: 4000000000, entity-tag: entity3}
+ - {id: a77d430237b307e6, ts: 4000000000, entity-tag: entity2}
+ - {id: ee7276ee58e4421c, ts: 4000000000, entity-tag: entity1}
+ - {id: adc82a95b2af3502, ts: 5000000000, entity-tag: entity1}
+ - {id: 14f994dd7433eaca, ts: 5000000000, entity-tag: entity3}
+ - {id: f0397cf91106fb01, ts: 5000000000, entity-tag: entity5}
+ - {id: 93ae3f61954ee5de, ts: 5000000000, entity-tag: entity4}
+ - {id: 7d4be3cf97db3464, ts: 5000000000, entity-tag: entity2}
diff --git a/banyand/stream/tracing.go b/banyand/stream/tracing.go
index 5bcddeccd..08250d5f2 100644
--- a/banyand/stream/tracing.go
+++ b/banyand/stream/tracing.go
@@ -18,18 +18,10 @@
package stream
import (
- "context"
"fmt"
"time"
"github.com/dustin/go-humanize"
-
- "github.com/apache/skywalking-banyandb/pkg/query"
-)
-
-const (
- partMetadataHeader = "MinTimestamp, MaxTimestamp, CompressionSize, UncompressedSize, TotalCount, BlocksCount"
- blockHeader = "PartID, SeriesID, MinTimestamp, MaxTimestamp, Count, UncompressedSize"
)
func (pm *partMetadata) String() string {
@@ -49,30 +41,3 @@ func (bc *blockCursor) String() string {
return fmt.Sprintf("%d, %d, %s, %s, %d, %s",
bc.p.partMetadata.ID, bc.bm.seriesID, minTimestamp, maxTimestamp, bc.bm.count, humanize.Bytes(bc.bm.uncompressedSizeBytes))
}
-
-func startBlockScanSpan(ctx context.Context, sids int, parts []*part, qr *idxResult) func() {
- tracer := query.GetTracer(ctx)
- if tracer == nil {
- return func() {}
- }
-
- span, _ := tracer.StartSpan(ctx, "scan-blocks")
- if qr.qo.elementFilter != nil {
- span.Tag("filter_size", fmt.Sprintf("%d", qr.qo.elementFilter.Len()))
- }
- span.Tag("series_num", fmt.Sprintf("%d", sids))
- span.Tag("part_header", partMetadataHeader)
- span.Tag("part_num", fmt.Sprintf("%d", len(parts)))
- for i := range parts {
- span.Tag(fmt.Sprintf("part_%d_%s", parts[i].partMetadata.ID, parts[i].path),
- parts[i].partMetadata.String())
- }
-
- return func() {
- span.Tag("block_header", blockHeader)
- for i := range qr.data {
- span.Tag(fmt.Sprintf("block_%d", i), qr.data[i].String())
- }
- span.Stop()
- }
-}
diff --git a/banyand/stream/vectorized_flag_test.go b/banyand/stream/vectorized_flag_test.go
new file mode 100644
index 000000000..3f7e5b36e
--- /dev/null
+++ b/banyand/stream/vectorized_flag_test.go
@@ -0,0 +1,57 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ vstream "github.com/apache/skywalking-banyandb/pkg/query/vectorized/stream"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+func parseVectorizedFlags(t *testing.T, args ...string) vstream.VectorizedConfig {
+ t.Helper()
+ cfg := vstream.VectorizedConfig{}
+ flagS := run.NewFlagSet("stream-vec-test")
+ bindVectorizedFlags(flagS, &cfg)
+ require.NoError(t, flagS.Parse(args))
+ return cfg
+}
+
+func TestVectorizedEnabledFalseAbortsStartup(t *testing.T) {
+ cfg := parseVectorizedFlags(t, "--stream-vectorized-enabled=false")
+ err := cfg.Validate()
+ require.Error(t, err, "--stream-vectorized-enabled=false must abort startup, not silently run the vec path")
+ require.Contains(t, err.Error(), "apache/skywalking#13998")
+
+ // Both node roles must surface it from their own Validate, which is what run.Group
+ // calls before PreRun — otherwise the flag would be accepted and quietly ignored.
+ standaloneSvc := &standalone{root: t.TempDir(), option: option{vectorized: cfg}}
+ standaloneSvc.retentionConfig.Cooldown = time.Second
+ require.ErrorContains(t, standaloneSvc.Validate(), "apache/skywalking#13998")
+ liaisonSvc := &liaison{root: t.TempDir(), option: option{vectorized: cfg}}
+ require.ErrorContains(t, liaisonSvc.Validate(), "apache/skywalking#13998")
+}
+
+func TestVectorizedEnabledTrueAndDefaultAreAccepted(t *testing.T) {
+ require.NoError(t, parseVectorizedFlags(t, "--stream-vectorized-enabled=true").Validate())
+ require.NoError(t, parseVectorizedFlags(t).Validate())
+}
diff --git a/banyand/trace/query.go b/banyand/trace/query.go
index e2bf39b10..a1b6035f0 100644
--- a/banyand/trace/query.go
+++ b/banyand/trace/query.go
@@ -196,46 +196,25 @@ func (t *trace) Query(ctx context.Context, tqo model.TraceQueryOptions) (model.T
pipelineCtx, cancel := context.WithTimeout(ctx, queryTimeout)
result.ctx = pipelineCtx
result.cancel = cancel
- result.recordCursor, result.recordResult, result.finishResultSpan = startQueryResultSpan(pipelineCtx)
-
- if result.keys == nil {
- result.keys = make(map[string]int64)
+ _, result.recordResult, result.finishResultSpan = startQueryResultSpan(pipelineCtx)
+
+ // Assign errors to err so the deferred result.Release() fires on failure,
+ // releasing segments, the timeout context, and the tracing span.
+ var vectorizedScanBatch *scanBatch
+ if vectorizedScanBatch, err = t.buildConsistentVectorizedScanBatch(
+ pipelineCtx, tables, qo, sidxInstances, sidxQueryRequest, useSIDXStreaming, tqo.MaxTraceSize,
+ ); err != nil {
+ return nil, err
}
-
- switch {
- case t.vectorized.Enabled:
- // Assign errors to err so the deferred result.Release() fires on failure,
- // releasing segments, the timeout context, and the tracing span.
- var vectorizedScanBatch *scanBatch
- if vectorizedScanBatch, err = t.buildConsistentVectorizedScanBatch(
- pipelineCtx, tables, qo, sidxInstances, sidxQueryRequest, useSIDXStreaming, tqo.MaxTraceSize,
- ); err != nil {
- return nil, err
- }
- var vectorizedResult *vectorizedTraceQueryResult
- vectorizedResult, err = newVectorizedTraceQueryResult(
- pipelineCtx, vectorizedScanBatch, qo, segments, cancel, result.finishResultSpan, result.recordResult)
- if err != nil {
- return nil, err
- }
- vtrace.IncrQueryCount()
- traceQueryResultTracker.Acquire(vectorizedResult)
- return vectorizedResult, nil
- case len(qo.traceIDs) > 0:
- traceBatchCh := staticTraceBatchSource(pipelineCtx, qo.traceIDs, tqo.MaxTraceSize, result.keys)
- result.cursorBatchCh = t.startBlockScanStage(pipelineCtx, tables, qo, traceBatchCh)
- case useSIDXStreaming:
- var streamDone <-chan struct{}
- traceBatchCh, streamDone := t.streamSIDXTraceBatches(pipelineCtx, sidxInstances, sidxQueryRequest, tqo.MaxTraceSize)
- result.streamDone = streamDone
- result.cursorBatchCh = t.startBlockScanStage(pipelineCtx, tables, qo, traceBatchCh)
- default:
- err = errors.New("invalid query options: either traceIDs or order must be specified")
- return nilResult, err
+ var vectorizedResult *vectorizedTraceQueryResult
+ vectorizedResult, err = newVectorizedTraceQueryResult(
+ pipelineCtx, vectorizedScanBatch, qo, segments, cancel, result.finishResultSpan, result.recordResult)
+ if err != nil {
+ return nil, err
}
-
- traceQueryResultTracker.Acquire(&result)
- return &result, nil
+ vtrace.IncrQueryCount()
+ traceQueryResultTracker.Acquire(vectorizedResult)
+ return vectorizedResult, nil
}
func omitIdentityTagProjection(projection *model.TagProjection, traceIDTagName, spanIDTagName string) *model.TagProjection {
diff --git a/banyand/trace/query_capacity_test.go b/banyand/trace/query_capacity_test.go
deleted file mode 100644
index bcd88459b..000000000
--- a/banyand/trace/query_capacity_test.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with this work for additional
-// information regarding copyright ownership. The ASF licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-package trace
-
-import (
- "math"
- "strconv"
- "testing"
-)
-
-// Trace queries previously turned limit(+offset) into MaxTraceSize / MaxBatchSize
-// and sized the legacy keys map from that value. newTraceBatch must keep the
-// initial map hint bounded for oversized requests.
-func TestTraceBatchCapacityCapsOversizedLimit(t *testing.T) {
- if got := traceBatchCapacity(math.MaxInt32); got != 1024 {
- t.Fatalf("traceBatchCapacity(MaxInt32)=%d, want 1024", got)
- }
- if got := traceBatchCapacity(16); got != 16 {
- t.Fatalf("traceBatchCapacity(16)=%d, want 16", got)
- }
- // Modest oversized input: a guard regression fails the helper assert above /
- // would only allocate a few KiB here, never MaxInt32 map buckets.
- const oversized = 4096
- if got := traceBatchCapacity(oversized); got != 1024 {
- t.Fatalf("traceBatchCapacity(%d)=%d, want 1024", oversized, got)
- }
- batch := newTraceBatch(0, oversized)
- if batch.keys == nil {
- t.Fatal("expected keys map for positive capped capacity")
- }
- for i := 0; i < 1025; i++ {
- batch.keys[strconv.Itoa(i)] = int64(i)
- }
-}
diff --git a/banyand/trace/query_vectorized_test.go b/banyand/trace/query_vectorized_test.go
index b6cd6a79a..bd7bd1bc7 100644
--- a/banyand/trace/query_vectorized_test.go
+++ b/banyand/trace/query_vectorized_test.go
@@ -72,7 +72,7 @@ func TestDrainVectorizedPhase1(t *testing.T) {
}
func TestBuildVectorizedPhase1TraceBatchStatic(t *testing.T) {
- tr := &trace{vectorized: vtrace.VectorizedConfig{Enabled: true, BatchSize: 1, QueryMemoryMiB: 1}}
+ tr := &trace{vectorized: vtrace.VectorizedConfig{BatchSize: 1, QueryMemoryMiB: 1}}
batch, err := tr.buildVectorizedPhase1TraceBatch(context.Background(), queryOptions{
traceIDs: []string{"trace-a", "trace-b"},
}, nil, sidx.QueryRequest{}, false, 1)
@@ -88,7 +88,7 @@ func TestBuildVectorizedPhase1TraceBatchStatic(t *testing.T) {
// Push path: traceIDs[:limit] → emit first occurrence only.
// Vectorized (fixed): same — truncate first, then dedup within the window.
func TestBuildVectorizedPhase1TraceBatchStaticDedupLimitParity(t *testing.T) {
- tr := &trace{vectorized: vtrace.VectorizedConfig{Enabled: true, BatchSize: 10, QueryMemoryMiB: 1}}
+ tr := &trace{vectorized: vtrace.VectorizedConfig{BatchSize: 10, QueryMemoryMiB: 1}}
// [a, a, b] limit 2: push takes first 2 raw = [a, a], emits a once.
// Vectorized must do the same: truncate to [a, a], dedup → [a].
batch, err := tr.buildVectorizedPhase1TraceBatch(context.Background(), queryOptions{
diff --git a/banyand/trace/streaming_pipeline.go b/banyand/trace/streaming_pipeline.go
index c3d236959..163140015 100644
--- a/banyand/trace/streaming_pipeline.go
+++ b/banyand/trace/streaming_pipeline.go
@@ -18,23 +18,10 @@
package trace
import (
- "container/heap"
"context"
- stdErrors "errors"
"fmt"
- "slices"
- "sort"
- "sync"
- "sync/atomic"
pkgerrors "github.com/pkg/errors"
-
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/internal/sidx"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/query"
- "github.com/apache/skywalking-banyandb/pkg/run"
)
type traceBatch struct {
@@ -53,806 +40,11 @@ type scanBatch struct {
traceBatch
}
-func newTraceBatch(seq int, capacity int) traceBatch {
- tb := traceBatch{
- seq: seq,
- traceIDs: make(map[uint64][]string),
- }
- // Keep the initial map allocation bounded; the batch remains semantically
- // bounded by its caller while large requests cannot force a giant allocation.
- capacity = traceBatchCapacity(capacity)
- if capacity > 0 {
- tb.keys = make(map[string]int64, capacity)
- }
- return tb
-}
-
-func traceBatchCapacity(capacity int) int {
- if capacity > 1024 {
- return 1024
- }
- if capacity < 0 {
- return 0
- }
- return capacity
-}
-
-func staticTraceBatchSource(ctx context.Context, traceIDs []string, maxTraceSize int, keys map[string]int64) <-chan traceBatch {
- out := make(chan traceBatch)
- batchSourceLogger := logger.GetLogger("trace-batch-source")
- run.Go(ctx, "trace-batch-source", batchSourceLogger, func(_ context.Context) {
- defer close(out)
-
- if len(traceIDs) == 0 {
- return
- }
-
- limit := len(traceIDs)
- if maxTraceSize > 0 && maxTraceSize < limit {
- limit = maxTraceSize
- }
-
- var retainedBytes uint64
- for _, traceID := range traceIDs[:limit] {
- retainedBytes += uint64(len(traceID)) + 64
- }
- if chargeErr := query.Charge(ctx, retainedBytes); chargeErr != nil {
- select {
- case out <- traceBatch{err: chargeErr}:
- case <-ctx.Done():
- }
- return
- }
- orderedIDs := append([]string(nil), traceIDs[:limit]...)
-
- // Determine batch size. When maxTraceSize is zero, emit everything in one batch.
- batchSize := maxTraceSize
- if batchSize <= 0 || batchSize > len(orderedIDs) {
- batchSize = len(orderedIDs)
- }
-
- seq := 0
- for start := 0; start < len(orderedIDs); start += batchSize {
- end := start + batchSize
- if end > len(orderedIDs) {
- end = len(orderedIDs)
- }
-
- // Static sources don't have partID information, use sentinel value 0
- batchIDs := append([]string(nil), orderedIDs[start:end]...)
- traceIDMap := make(map[uint64][]string)
- traceIDMap[0] = batchIDs
-
- select {
- case <-ctx.Done():
- return
- case out <- traceBatch{
- seq: seq,
- traceIDs: traceIDMap,
- traceIDsOrder: batchIDs,
- keys: keys,
- }:
- seq++
- }
- }
- })
-
- return out
-}
-
-const defaultTraceBatchSize = 64
-
-type sidxStreamShard struct {
- results <-chan *sidx.QueryResponse
- response *sidx.QueryResponse
- id int
- idx int
- done bool
-}
-
-func (sh *sidxStreamShard) prepare(ctx context.Context) error {
- for {
- if sh.response != nil && sh.idx >= 0 && sh.idx < sh.response.Len() {
- return nil
- }
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case resp, ok := <-sh.results:
- if !ok {
- sh.done = true
- sh.response = nil
- return nil
- }
- if resp == nil {
- continue
- }
- if resp.Error != nil {
- return resp.Error
- }
- if resp.Len() == 0 {
- continue
- }
- sh.response = resp
- sh.idx = 0
- return nil
- }
- }
-}
-
-func (sh *sidxStreamShard) currentKey() int64 {
- return sh.response.Keys[sh.idx]
-}
-
-func (sh *sidxStreamShard) currentData() []byte {
- return sh.response.Data[sh.idx]
-}
-
-func (sh *sidxStreamShard) advance(ctx context.Context) error {
- if sh.response == nil {
- return sh.prepare(ctx)
- }
-
- sh.idx++
-
- return sh.prepare(ctx)
-}
-
-type sidxStreamHeap struct {
- shards []*sidxStreamShard
- asc bool
-}
-
-func (h sidxStreamHeap) Len() int {
- return len(h.shards)
-}
-
-func (h sidxStreamHeap) Less(i, j int) bool {
- left := h.shards[i].currentKey()
- right := h.shards[j].currentKey()
- if h.asc {
- return left < right
- }
- return left > right
-}
-
-func (h sidxStreamHeap) Swap(i, j int) {
- h.shards[i], h.shards[j] = h.shards[j], h.shards[i]
-}
-
-func (h *sidxStreamHeap) Push(x interface{}) {
- h.shards = append(h.shards, x.(*sidxStreamShard))
-}
-
-func (h *sidxStreamHeap) Pop() interface{} {
- n := len(h.shards)
- x := h.shards[n-1]
- h.shards = h.shards[:n-1]
- return x
-}
-
-type sidxStreamError struct {
- err error
- index int
-}
-
-func decodeTraceID(data []byte) (string, error) {
- if len(data) == 0 {
- return "", fmt.Errorf("empty trace id payload")
- }
- if idFormat(data[0]) != idFormatV1 {
- return "", fmt.Errorf("invalid trace id format: %x", data[0])
- }
- return string(data[1:]), nil
-}
-
-func forwardSIDXError(ctx context.Context, idx int, errCh <-chan error, out chan<- sidxStreamError) {
- if errCh == nil {
- return
- }
- for err := range errCh {
- if err == nil {
- continue
- }
- select {
- case out <- sidxStreamError{index: idx, err: err}:
- case <-ctx.Done():
- select {
- case out <- sidxStreamError{index: idx, err: err}:
- default:
- }
- }
- return
- }
-}
-
-func (t *trace) streamSIDXTraceBatches(
- ctx context.Context,
- sidxInstances []sidx.SIDX,
- req sidx.QueryRequest,
- maxTraceSize int,
-) (<-chan traceBatch, <-chan struct{}) {
- out := make(chan traceBatch)
- done := make(chan struct{})
-
- if len(sidxInstances) == 0 {
- close(out)
- close(done)
- return out, done
- }
-
- tracer := query.GetTracer(ctx)
- tracingCtx := ctx
- var span *query.Span
- if tracer != nil {
- span, tracingCtx = tracer.StartSpan(ctx, "sidx-stream")
- tagSIDXStreamSpan(span, req, maxTraceSize, len(sidxInstances))
- }
- streamCtx, cancel := context.WithCancel(tracingCtx)
- runner := newSIDXStreamRunner(ctx, streamCtx, cancel, req, maxTraceSize)
- run.Go(tracingCtx, "trace-sidx-stream", t.l, func(_ context.Context) {
- defer func() {
- if span != nil {
- span.Tagf("batches_emitted", "%d", runner.batchesEmitted.Load())
- span.Tagf("trace_ids_emitted", "%d", runner.total.Load())
- if dups := runner.duplicates.Load(); dups > 0 {
- span.Tagf("duplicate_trace_ids", "%d", dups)
- }
- if spanErr := runner.getSpanErr(); spanErr != nil && !stdErrors.Is(spanErr, context.Canceled) {
- span.Error(spanErr)
- }
- span.Stop()
- }
- cancel()
- close(out)
- close(done)
- }()
-
- if prepareErr := runner.prepare(sidxInstances); prepareErr != nil {
- runner.cancel()
- runner.emitError(out, prepareErr)
- return
- }
-
- runner.run(out)
- })
- return out, done
-}
-
-type sidxStreamRunner struct {
- streamCtx context.Context
- ctx context.Context
- spanErr atomic.Value // stores error
- cancelFunc context.CancelFunc
- errEvents chan sidxStreamError
- heap *sidxStreamHeap
- seenTraceIDs map[string]struct{}
- batch traceBatch
- req sidx.QueryRequest
- errWg sync.WaitGroup
- nextSeq int
- total atomic.Int64
- batchesEmitted atomic.Int64
- duplicates atomic.Int64
- batchSize int
-}
-
-func tagSIDXStreamSpan(span *query.Span, req sidx.QueryRequest, maxTraceSize int, instanceCount int) {
- if span == nil {
- return
- }
- if req.Filter != nil {
- span.Tag("filter_present", "true")
- } else {
- span.Tag("filter_present", "false")
- }
- if req.Order != nil {
- if req.Order.Index != nil && req.Order.Index.GetMetadata() != nil {
- span.Tag("order_index", req.Order.Index.GetMetadata().GetName())
- }
- span.Tag("order_sort", req.Order.Sort.String())
- span.Tagf("order_type", "%d", req.Order.Type)
- } else {
- span.Tag("order_sort", "none")
- }
- span.Tagf("series_id_candidates", "%d", len(req.SeriesIDs))
- span.Tagf("max_batch_size", "%d", req.MaxBatchSize)
- span.Tagf("max_trace_size", "%d", maxTraceSize)
- span.Tagf("sidx_instance_count", "%d", instanceCount)
-}
-
-func newSIDXStreamRunner(
- ctx context.Context,
- streamCtx context.Context,
- cancel context.CancelFunc,
- req sidx.QueryRequest,
- maxTraceSize int,
-) *sidxStreamRunner {
- asc := true
- if req.Order != nil && req.Order.Sort == modelv1.Sort_SORT_DESC {
- asc = false
- }
-
- batchSize := req.MaxBatchSize
- if batchSize <= 0 {
- if maxTraceSize > 0 {
- batchSize = maxTraceSize
- } else {
- batchSize = defaultTraceBatchSize
- }
- }
-
- return &sidxStreamRunner{
- ctx: ctx,
- streamCtx: streamCtx,
- cancelFunc: cancel,
- req: req,
- batchSize: batchSize,
- heap: &sidxStreamHeap{asc: asc},
- seenTraceIDs: make(map[string]struct{}),
- batch: newTraceBatch(0, batchSize),
- nextSeq: 1,
- }
-}
-
-func (r *sidxStreamRunner) prepare(instances []sidx.SIDX) error {
- type shardSource struct {
- shard *sidxStreamShard
- errCh <-chan error
- idx int
- }
-
- // Track all error channels separately from shards with data
- type errChannelInfo struct {
- errCh <-chan error
- idx int
- }
- allErrChannels := make([]errChannelInfo, 0, len(instances))
-
- sources := make([]shardSource, 0, len(instances))
- for idx, instance := range instances {
- resultsCh, errCh := instance.StreamingQuery(r.streamCtx, r.req)
-
- // Track error channel regardless of whether shard has data
- if errCh != nil {
- allErrChannels = append(allErrChannels, errChannelInfo{errCh: errCh, idx: idx})
- }
-
- shard := &sidxStreamShard{
- id: idx,
- results: resultsCh,
- }
- if err := shard.prepare(r.streamCtx); err != nil {
- return fmt.Errorf("sidx[%d] prepare failed: %w", idx, err)
- }
- if shard.done {
- // Shard has no data, but we still track its error channel above
- continue
- }
- sources = append(sources, shardSource{
- shard: shard,
- errCh: errCh,
- idx: idx,
- })
- }
-
- // Add shards with data to the heap
- for _, src := range sources {
- heap.Push(r.heap, src.shard)
- }
-
- // Create error event channel and start error forwarding goroutines
- // for ALL error channels, even from shards without data
- if len(allErrChannels) > 0 {
- r.errEvents = make(chan sidxStreamError, len(allErrChannels))
- errFwdLogger := logger.GetLogger("trace-sidx-error-forward")
- for _, errSrc := range allErrChannels {
- r.errWg.Add(1)
- errIdx, errCh := errSrc.idx, errSrc.errCh
- run.Go(r.streamCtx, "trace-sidx-error-forward", errFwdLogger, func(runCtx context.Context) {
- defer r.errWg.Done()
- forwardSIDXError(runCtx, errIdx, errCh, r.errEvents)
- })
- }
-
- go func() {
- r.errWg.Wait()
- close(r.errEvents)
- }()
- }
-
- return nil
-}
-
-func (r *sidxStreamRunner) run(out chan<- traceBatch) {
- // Always drain error events before returning, even on early exit
- // Cancel first to stop SIDX goroutines, then drain any pending errors
- defer func() {
- r.cancel()
- r.drainErrorEvents(out)
- }()
-
- if r.heap.Len() == 0 {
- return
- }
-
- for r.heap.Len() > 0 {
- if err := r.streamCtx.Err(); err != nil {
- r.recordSpanErr(err)
- return
- }
-
- if !r.pollErrEvents(out) {
- return
- }
-
- shard := heap.Pop(r.heap).(*sidxStreamShard)
- if err := r.ensureShardReady(shard); err != nil {
- r.emitError(out, err)
- return
- }
- if shard.done {
- continue
- }
-
- added, err := r.consumeShard(shard)
- if err != nil {
- r.emitError(out, err)
- return
- }
-
- if added {
- // Count total trace IDs across all partIDs
- totalTraceIDs := 0
- for _, ids := range r.batch.traceIDs {
- totalTraceIDs += len(ids)
- }
- if totalTraceIDs >= r.batchSize {
- if !r.emitBatch(out) {
- return
- }
- }
- }
-
- if err := r.advanceShard(shard); err != nil {
- r.emitError(out, err)
- return
- }
- }
-
- // Emit remaining batch if it has any trace IDs
- hasTraceIDs := false
- for _, ids := range r.batch.traceIDs {
- if len(ids) > 0 {
- hasTraceIDs = true
- break
- }
- }
- if hasTraceIDs {
- if !r.emitBatch(out) {
- return
- }
- }
-}
-
-func (r *sidxStreamRunner) pollErrEvents(out chan<- traceBatch) bool {
- if r.errEvents == nil {
- return true
- }
-
- // Check for errors without blocking
- select {
- case ev, ok := <-r.errEvents:
- if !ok {
- r.errEvents = nil
- return true
- }
- if ev.err == nil {
- return true
- }
- eventErr := fmt.Errorf("sidx[%d] streaming error: %w", ev.index, ev.err)
- r.emitError(out, eventErr)
- return false
- default:
- // No error available yet, continue processing
- return true
- }
-}
-
-func (r *sidxStreamRunner) ensureShardReady(shard *sidxStreamShard) error {
- if shard.response == nil {
- if err := shard.prepare(r.streamCtx); err != nil {
- return fmt.Errorf("sidx[%d] prepare failed: %w", shard.id, err)
- }
- if shard.done {
- return nil
- }
- }
- return nil
-}
-
-func (r *sidxStreamRunner) consumeShard(shard *sidxStreamShard) (bool, error) {
- traceID, err := decodeTraceID(shard.currentData())
- if err != nil {
- return false, fmt.Errorf("sidx[%d] invalid trace id payload: %w", shard.id, err)
- }
-
- if _, exists := r.seenTraceIDs[traceID]; exists {
- r.duplicates.Add(1)
- return false, nil
- }
- if chargeErr := query.Charge(r.streamCtx, uint64(len(traceID))+128); chargeErr != nil {
- return false, chargeErr
- }
-
- // Extract partID from SIDX response
- partID := shard.response.PartIDs[shard.idx]
-
- r.seenTraceIDs[traceID] = struct{}{}
- r.batch.traceIDs[partID] = append(r.batch.traceIDs[partID], traceID)
- r.batch.traceIDsOrder = append(r.batch.traceIDsOrder, traceID)
- if r.batch.keys == nil {
- r.batch.keys = make(map[string]int64)
- }
- r.batch.keys[traceID] = shard.currentKey()
- r.total.Add(1)
-
- return true, nil
-}
-
-func (r *sidxStreamRunner) emitBatch(out chan<- traceBatch) bool {
- select {
- case <-r.streamCtx.Done():
- r.recordSpanErr(r.streamCtx.Err())
- return false
- case out <- r.batch:
- r.batchesEmitted.Add(1)
- r.batch = newTraceBatch(r.nextSeq, r.batchSize)
- r.nextSeq++
- return true
- }
-}
-
-func (r *sidxStreamRunner) advanceShard(shard *sidxStreamShard) error {
- if err := shard.advance(r.streamCtx); err != nil {
- return fmt.Errorf("sidx[%d] advance failed: %w", shard.id, err)
- }
- if !shard.done {
- heap.Push(r.heap, shard)
- }
- return nil
-}
-
-func (r *sidxStreamRunner) emitError(out chan<- traceBatch, err error) {
- r.recordSpanErr(err)
- // Always try to send error even if context is canceled
- // The receiver needs to know about the error
- select {
- case out <- traceBatch{err: err}:
- case <-r.ctx.Done():
- // Context canceled, but still try to send error with non-blocking attempt
- select {
- case out <- traceBatch{err: err}:
- default:
- // Channel might be blocked or closed, can't send error
- }
- }
-}
-
-func (r *sidxStreamRunner) drainErrorEvents(out chan<- traceBatch) {
- if r.errEvents == nil {
- return
- }
-
- // Read from errEvents until it's closed or we find an error
- // The channel will be closed after all error forwarding goroutines finish
- for {
- select {
- case ev, ok := <-r.errEvents:
- if !ok {
- // Channel closed, all error forwarding goroutines finished
- return
- }
- if ev.err == nil {
- // No error in this event, continue reading
- continue
- }
- // Skip context.Canceled errors if we're shutting down normally
- // (these are expected cleanup errors, not actual failures)
- if stdErrors.Is(ev.err, context.Canceled) {
- continue
- }
- // Found a real error, emit it and return
- eventErr := fmt.Errorf("sidx[%d] streaming error: %w", ev.index, ev.err)
- r.emitError(out, eventErr)
- return
- case <-r.ctx.Done():
- // Context canceled, but still do non-blocking drain
- // to catch any errors that arrived before cancellation
- for {
- select {
- case ev, ok := <-r.errEvents:
- if !ok {
- return
- }
- if ev.err != nil && !stdErrors.Is(ev.err, context.Canceled) {
- eventErr := fmt.Errorf("sidx[%d] streaming error: %w", ev.index, ev.err)
- r.emitError(out, eventErr)
- return
- }
- default:
- // No more errors available
- r.recordSpanErr(r.ctx.Err())
- return
- }
- }
- }
- }
-}
-
-func (r *sidxStreamRunner) recordSpanErr(err error) {
- // Only store the first error
- if r.spanErr.Load() == nil {
- r.spanErr.Store(err)
- }
-}
-
-func (r *sidxStreamRunner) getSpanErr() error {
- if v := r.spanErr.Load(); v != nil {
- return v.(error)
- }
- return nil
-}
-
-func (r *sidxStreamRunner) cancel() {
- if r.cancelFunc != nil {
- r.cancelFunc()
- }
-}
-
-func (t *trace) startBlockScanStage(
- ctx context.Context,
- tables []*tsTable,
- qo queryOptions,
- batches <-chan traceBatch,
-) <-chan *scanBatch {
- out := make(chan *scanBatch)
-
- run.Go(ctx, "trace-block-scan", t.l, func(_ context.Context) {
- defer close(out)
-
- for batch := range batches {
- if batch.err != nil {
- select {
- case <-ctx.Done():
- return
- case out <- &scanBatch{traceBatch: batch, err: batch.err}:
- }
- continue
- }
-
- // Acquire snapshots from all tables
- snapshots := make([]*snapshot, 0, len(tables))
- for _, table := range tables {
- s := table.currentSnapshot()
- if s == nil {
- continue
- }
- snapshots = append(snapshots, s)
- }
-
- // If no snapshots available, skip this batch
- if len(snapshots) == 0 {
- select {
- case <-ctx.Done():
- return
- case out <- &scanBatch{traceBatch: batch}:
- }
- continue
- }
-
- // Start part selection span
- partSelectionCtx, finishPartSelection := startPartSelectionSpan(ctx, &batch, snapshots)
-
- parts := make([]*part, 0)
- groupedIDs := make([][]string, 0)
-
- allTraceIDs := make([]string, 0)
- for _, ids := range batch.traceIDs {
- allTraceIDs = append(allTraceIDs, ids...)
- }
- sort.Strings(allTraceIDs)
-
- bloomFilteredPartIDs := make([]uint64, 0)
- totalGroupedIDs := 0
-
- for _, s := range snapshots {
- for _, pw := range s.parts {
- p := pw.p
- partID := p.partMetadata.ID
-
- var idsFromSIDX []string
- if traceIDsFromSIDX, exists := batch.traceIDs[partID]; exists {
- idsFromSIDX = append([]string(nil), traceIDsFromSIDX...)
- }
- var idsForPart []string
- for _, traceID := range allTraceIDs {
- if slices.Contains(idsFromSIDX, traceID) || p.traceIDFilter.filter.MightContain(convert.StringToBytes(traceID)) {
- idsForPart = append(idsForPart, traceID)
- }
- }
-
- if len(idsForPart) > 0 {
- parts = append(parts, p)
- groupedIDs = append(groupedIDs, idsForPart)
- totalGroupedIDs += len(idsForPart)
- } else {
- bloomFilteredPartIDs = append(bloomFilteredPartIDs, partID)
- }
- }
- }
-
- if finishPartSelection != nil {
- finishPartSelection(&partSelectionMetrics{
- bloomFilteredPartIDs: bloomFilteredPartIDs,
- totalGroupedIDs: totalGroupedIDs,
- }, len(parts))
- }
-
- // Create the cursor channel and scanBatch
- cursorCh := make(chan scanCursorResult)
- sb := &scanBatch{
- traceBatch: batch,
- cursorCh: cursorCh,
- snapshots: snapshots,
- }
-
- // Send batch downstream first so consumer can start reading
- select {
- case <-ctx.Done():
- close(cursorCh)
- // Release snapshots on early exit
- for _, s := range snapshots {
- s.decRef()
- }
- return
- case out <- sb:
- }
-
- // Now scan inline and populate the channel
- t.scanPartsInline(partSelectionCtx, parts, groupedIDs, qo, cursorCh)
- close(cursorCh)
- }
- })
-
- return out
-}
-
type scanCursorResult struct {
cursor *blockCursor
err error
}
-func (t *trace) scanPartsInline(ctx context.Context, parts []*part, groupedIDs [][]string, qo queryOptions, out chan<- scanCursorResult) {
- cursors, scanErr := t.scanPartsInlineSync(ctx, parts, groupedIDs, qo)
- if scanErr != nil {
- select {
- case out <- scanCursorResult{err: scanErr}:
- case <-ctx.Done():
- }
- return
- }
- for _, cursor := range cursors {
- select {
- case out <- scanCursorResult{cursor: cursor}:
- case <-ctx.Done():
- releaseBlockCursor(cursor)
- return
- }
- }
-}
-
func (t *trace) scanPartsInlineSync(ctx context.Context, parts []*part, groupedIDs [][]string, qo queryOptions) ([]*blockCursor, error) {
if len(parts) == 0 {
return nil, nil
@@ -903,26 +95,21 @@ func (t *trace) scanPartsInlineSync(ctx context.Context, parts []*part, groupedI
}
hit++
- p := tstIter.piPool[tstIter.idx]
- blockSize := p.curBlock.uncompressedSpanSizeBytes
- if chargeErr := query.Charge(ctx, blockSize); chargeErr != nil {
- spanErr = chargeErr
- for _, cursor := range cursors {
- releaseBlockCursor(cursor)
- }
- return nil, spanErr
- }
- // Create block cursor only after charge and quota checks.
+ // Create block cursor and get size before checking quota
bc := generateBlockCursor()
+ p := tstIter.piPool[tstIter.idx]
bc.init(p.p, p.curBlock, qo)
+ blockSize := bc.bm.uncompressedSpanSizeBytes
// Check if adding this block would exceed quota
if quota >= 0 && spanBlockBytes+blockSize > uint64(quota) {
releaseBlockCursor(bc)
- spanErr = fmt.Errorf("block scan quota exceeded: block size %d bytes, quota is %d bytes", blockSize, quota)
- for _, cursor := range cursors {
- releaseBlockCursor(cursor)
+ if cursorCount > 0 {
+ // Have results, return them successfully by just closing channel
+ return cursors, nil
}
+ // No results, send error
+ spanErr = fmt.Errorf("block scan quota exceeded: block size %d bytes, quota is %d bytes", blockSize, quota)
return nil, spanErr
}
diff --git a/banyand/trace/streaming_pipeline_test.go b/banyand/trace/streaming_pipeline_test.go
index fccc6ec5c..5d23451a0 100644
--- a/banyand/trace/streaming_pipeline_test.go
+++ b/banyand/trace/streaming_pipeline_test.go
@@ -19,44 +19,11 @@ package trace
import (
"context"
- "errors"
- "strconv"
- "testing"
- "github.com/google/go-cmp/cmp"
-
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/internal/sidx"
"github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/pkg/index"
- "github.com/apache/skywalking-banyandb/pkg/query"
)
-type traceTinyLease struct{ remaining uint64 }
-
-func (l *traceTinyLease) Charge(bytes uint64) error {
- if bytes > l.remaining {
- return errors.New("trace test budget exhausted")
- }
- l.remaining -= bytes
- return nil
-}
-func (*traceTinyLease) Limit() uint64 { return 1024 }
-func (l *traceTinyLease) Used() uint64 { return 1024 - l.remaining }
-func (*traceTinyLease) Owner() any { return "trace-test" }
-
-func TestStaticTraceBatchSourceReportsChargeFailure(t *testing.T) {
- ctx := query.WithBudgetLease(context.Background(), &traceTinyLease{remaining: 1})
- ids := []string{string(make([]byte, 4096))}
- batch, ok := <-staticTraceBatchSource(ctx, ids, 0, nil)
- if !ok {
- t.Fatal("source closed without reporting charge failure")
- }
- if batch.err == nil {
- t.Fatal("expected charge failure")
- }
-}
-
type fakeSIDX struct {
responses []*sidx.QueryResponse
}
@@ -138,840 +105,9 @@ func (f *fakeSIDX) PrepareSynced(map[uint64]struct{}) func(cur *sidx.Snapshot) *
func (f *fakeSIDX) CurrentSnapshot() *sidx.Snapshot { return nil }
func (f *fakeSIDX) ReplaceSnapshot(*sidx.Snapshot) {}
-type fakeSIDXWithErr struct {
- *fakeSIDX
- err error
-}
-
-func (f *fakeSIDXWithErr) StreamingQuery(ctx context.Context, _ sidx.QueryRequest) (<-chan *sidx.QueryResponse, <-chan error) {
- results := make(chan *sidx.QueryResponse, len(f.responses))
- errCh := make(chan error, 1)
-
- go func() {
- defer close(results)
- defer close(errCh)
-
- for _, resp := range f.responses {
- select {
- case <-ctx.Done():
- return
- case results <- resp:
- }
- }
-
- // Send error after all results have been sent
- // This tests that errors are propagated even after data processing
- if f.err != nil {
- select {
- case <-ctx.Done():
- case errCh <- f.err:
- }
- }
- }()
-
- return results, errCh
-}
-
func encodeTraceIDForTest(id string) []byte {
buf := make([]byte, len(id)+1)
buf[0] = byte(idFormatV1)
copy(buf[1:], id)
return buf
}
-
-func TestStreamSIDXTraceBatches_ProducesOrderedBatches(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 2,
- }
-
- responses := []*sidx.QueryResponse{
- {
- Keys: []int64{1, 2},
- Data: [][]byte{
- encodeTraceIDForTest("a"),
- encodeTraceIDForTest("b"),
- },
- PartIDs: []uint64{100, 101},
- },
- {
- Keys: []int64{2, 3},
- Data: [][]byte{
- encodeTraceIDForTest("b"),
- encodeTraceIDForTest("c"),
- },
- PartIDs: []uint64{101, 102},
- },
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{&fakeSIDX{responses: responses}}, req, 3)
-
- var batches []traceBatch
- for batch := range batchCh {
- if batch.err != nil {
- t.Fatalf("unexpected error batch: %v", batch.err)
- }
- batches = append(batches, batch)
- }
-
- if len(batches) != 2 {
- t.Fatalf("expected 2 batches, got %d", len(batches))
- }
-
- wantBatch0 := map[uint64][]string{
- 100: {"a"},
- 101: {"b"},
- }
- if diff := cmp.Diff(wantBatch0, batches[0].traceIDs); diff != "" {
- t.Fatalf("first batch mismatch (-want +got):\n%s", diff)
- }
-
- wantBatch1 := map[uint64][]string{
- 102: {"c"},
- }
- if diff := cmp.Diff(wantBatch1, batches[1].traceIDs); diff != "" {
- t.Fatalf("second batch mismatch (-want +got):\n%s", diff)
- }
-
- wantKeys := map[string]int64{"a": 1, "b": 2, "c": 3}
- for _, batch := range batches {
- for _, ids := range batch.traceIDs {
- for _, tid := range ids {
- if got := batch.keys[tid]; got != wantKeys[tid] {
- t.Fatalf("unexpected key for %s: got %d, want %d", tid, got, wantKeys[tid])
- }
- }
- }
- }
-}
-
-func TestStreamSIDXTraceBatches_OrdersDescending(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_DESC},
- MaxBatchSize: 2,
- }
-
- responses := []*sidx.QueryResponse{
- {
- Keys: []int64{5, 4},
- Data: [][]byte{
- encodeTraceIDForTest("e"),
- encodeTraceIDForTest("d"),
- },
- PartIDs: []uint64{200, 201},
- },
- {
- Keys: []int64{4, 3},
- Data: [][]byte{
- encodeTraceIDForTest("d"),
- encodeTraceIDForTest("c"),
- },
- PartIDs: []uint64{201, 202},
- },
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{&fakeSIDX{responses: responses}}, req, 3)
-
- var batches []traceBatch
- for batch := range batchCh {
- if batch.err != nil {
- t.Fatalf("unexpected error batch: %v", batch.err)
- }
- batches = append(batches, batch)
- }
-
- if len(batches) != 2 {
- t.Fatalf("expected 2 batches, got %d", len(batches))
- }
-
- wantBatch0 := map[uint64][]string{
- 200: {"e"},
- 201: {"d"},
- }
- if diff := cmp.Diff(wantBatch0, batches[0].traceIDs); diff != "" {
- t.Fatalf("first batch mismatch (-want +got):\n%s", diff)
- }
-
- wantBatch1 := map[uint64][]string{
- 202: {"c"},
- }
- if diff := cmp.Diff(wantBatch1, batches[1].traceIDs); diff != "" {
- t.Fatalf("second batch mismatch (-want +got):\n%s", diff)
- }
-
- wantKeys := map[string]int64{"e": 5, "d": 4, "c": 3}
- for _, batch := range batches {
- for _, ids := range batch.traceIDs {
- for _, tid := range ids {
- if got := batch.keys[tid]; got != wantKeys[tid] {
- t.Fatalf("unexpected key for %s: got %d, want %d", tid, got, wantKeys[tid])
- }
- }
- }
- }
-}
-
-func TestStreamSIDXTraceBatches_PropagatesErrorAfterCancellation(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 1,
- }
-
- streamErr := errors.New("stream failure")
-
- sidxInstance := &fakeSIDXWithErr{
- fakeSIDX: &fakeSIDX{
- responses: []*sidx.QueryResponse{
- {
- Keys: []int64{1},
- Data: [][]byte{
- encodeTraceIDForTest("trace-1"),
- },
- PartIDs: []uint64{300},
- },
- },
- },
- err: streamErr,
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0)
-
- var (
- dataSeen bool
- errBatch traceBatch
- errSeen bool
- )
-
- for batch := range batchCh {
- if batch.err != nil {
- errBatch = batch
- errSeen = true
- continue
- }
- for _, ids := range batch.traceIDs {
- if len(ids) > 0 {
- dataSeen = true
- break
- }
- }
- }
-
- // The error must be propagated
- if !errSeen {
- t.Fatalf("expected error batch but none received")
- }
- if !errors.Is(errBatch.err, streamErr) {
- t.Fatalf("unexpected error: %v", errBatch.err)
- }
-
- // Data may or may not be seen depending on error timing.
- // If the error arrives on errEvents before the first loop iteration processes
- // the heap, dataSeen will be false. Both outcomes are valid.
- // The important invariant is that the error is always propagated.
- if dataSeen {
- t.Logf("data was processed before error detected (typical case)")
- } else {
- t.Logf("error detected before data processing (race condition)")
- }
-}
-
-// fakeSIDXWithImmediateError simulates errors from blockScanResultBatch.
-// by sending errors immediately via errCh (like scan errors would).
-type fakeSIDXWithImmediateError struct {
- *fakeSIDX
- err error
-}
-
-func (f *fakeSIDXWithImmediateError) StreamingQuery(ctx context.Context, _ sidx.QueryRequest) (<-chan *sidx.QueryResponse, <-chan error) {
- results := make(chan *sidx.QueryResponse, len(f.responses))
- errCh := make(chan error, 1)
-
- go func() {
- defer close(results)
- defer close(errCh)
-
- // Send error immediately (simulating blockScanner.scan error)
- if f.err != nil {
- errCh <- f.err
- }
-
- // Then send any responses
- for _, resp := range f.responses {
- select {
- case <-ctx.Done():
- return
- case results <- resp:
- }
- }
- }()
-
- return results, errCh
-}
-
-// TestStreamSIDXTraceBatches_PropagatesBlockScannerError verifies that errors.
-// from blockScanResultBatch.err (sent via errCh) are properly propagated through
-// the streaming pipeline to the traceBatch channel and ultimately to queryResult.Pull().
-func TestStreamSIDXTraceBatches_PropagatesBlockScannerError(t *testing.T) {
- tests := []struct {
- name string
- scanError string
- errorContain string
- }{
- {
- name: "iterator_init_error",
- scanError: "cannot init iter: iterator initialization failed",
- errorContain: "cannot init iter",
- },
- {
- name: "block_scan_error",
- scanError: "sidx block scan quota exceeded: used 1000 bytes, quota is 500 bytes",
- errorContain: "quota exceeded",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 2,
- }
-
- scanErr := errors.New(tt.scanError)
-
- // Use fakeSIDXWithImmediateError - since errCh and results channel
- // race each other, we don't test ordering, just that error propagates
- sidxInstance := &fakeSIDXWithImmediateError{
- fakeSIDX: &fakeSIDX{
- responses: nil,
- },
- err: scanErr,
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0)
-
- var receivedError error
-
- // Consume all batches until we receive the error
- for batch := range batchCh {
- if batch.err != nil {
- receivedError = batch.err
- // Don't break - consume remaining batches to avoid goroutine leaks
- continue
- }
- }
-
- // Verify error was eventually propagated
- if receivedError == nil {
- t.Fatalf("expected error containing %q but got none", tt.errorContain)
- }
- if !errors.Is(receivedError, scanErr) {
- // Check if error is wrapped
- errMsg := receivedError.Error()
- if errMsg == "" || !contains(errMsg, tt.errorContain) {
- t.Fatalf("expected error containing %q but got: %v", tt.errorContain, receivedError)
- }
- }
- })
- }
-}
-
-// TestStreamSIDXTraceBatches_DrainErrorEventsGuaranteed verifies that.
-// drainErrorEvents is always called via defer, even on early returns.
-func TestStreamSIDXTraceBatches_DrainErrorEventsGuaranteed(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 10,
- }
-
- scanErr := errors.New("scan error from defer test")
-
- // Create fake that will send error after some results
- sidxInstance := &fakeSIDXWithImmediateError{
- fakeSIDX: &fakeSIDX{
- responses: []*sidx.QueryResponse{
- {
- Keys: []int64{1},
- Data: [][]byte{
- encodeTraceIDForTest("trace-1"),
- },
- PartIDs: []uint64{400},
- },
- },
- },
- err: scanErr,
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- // Cancel context immediately to force early return
- cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0)
-
- var receivedError error
-
- // Even with canceled context, we should receive the error
- // because defer ensures drainErrorEvents is called
- for batch := range batchCh {
- if batch.err != nil {
- receivedError = batch.err
- }
- }
-
- // The error might not be propagated if context is canceled immediately
- // but this test verifies the defer mechanism exists
- t.Logf("Received error (may be nil due to immediate cancellation): %v", receivedError)
-}
-
-// TestStreamSIDXTraceBatches_ErrorEmissionResilience verifies that.
-// emitError tries to send even when context is canceled.
-func TestStreamSIDXTraceBatches_ErrorEmissionResilience(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 1,
- }
-
- scanErr := errors.New("scan error during processing")
-
- sidxInstance := &fakeSIDXWithImmediateError{
- fakeSIDX: &fakeSIDX{
- responses: []*sidx.QueryResponse{
- {
- Keys: []int64{1, 2, 3},
- Data: [][]byte{
- encodeTraceIDForTest("trace-1"),
- encodeTraceIDForTest("trace-2"),
- encodeTraceIDForTest("trace-3"),
- },
- PartIDs: []uint64{500, 501, 502},
- },
- },
- },
- err: scanErr,
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0)
-
- var receivedError error
- batchCount := 0
-
- // Read first batch, then wait a bit for error to be sent to errEvents
- for batch := range batchCh {
- if batch.err != nil {
- receivedError = batch.err
- break
- }
- batchCount++
- // After first batch, continue reading to trigger drainErrorEvents
- }
-
- // We should eventually get the error either from pollErrEvents or drainErrorEvents
- if receivedError == nil {
- t.Fatal("expected error to be propagated but got none")
- }
-
- if !errors.Is(receivedError, scanErr) && !contains(receivedError.Error(), "scan error") {
- t.Fatalf("unexpected error: %v", receivedError)
- }
-}
-
-func contains(s, substr string) bool {
- return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr))
-}
-
-func containsHelper(s, substr string) bool {
- for i := 0; i <= len(s)-len(substr); i++ {
- if s[i:i+len(substr)] == substr {
- return true
- }
- }
- return false
-}
-
-// fakeSIDXInfinite simulates a SIDX that returns an infinite stream of results.
-// It continues to generate responses until the context is canceled.
-type fakeSIDXInfinite struct {
- traceIDPrefix string
- batchSize int
- keyStart int64
-}
-
-func (f *fakeSIDXInfinite) ScanQuery(context.Context, sidx.ScanQueryRequest) ([]*sidx.QueryResponse, error) {
- return nil, nil
-}
-
-func (f *fakeSIDXInfinite) QuerySync(_ context.Context, _ sidx.QueryRequest) ([]*sidx.QueryResponse, error) {
- return nil, nil
-}
-
-func (f *fakeSIDXInfinite) StreamingQuery(ctx context.Context, _ sidx.QueryRequest) (<-chan *sidx.QueryResponse, <-chan error) {
- results := make(chan *sidx.QueryResponse)
- errCh := make(chan error, 1)
-
- go func() {
- defer close(results)
- defer close(errCh)
-
- key := f.keyStart
- counter := 0
-
- for {
- select {
- case <-ctx.Done():
- errCh <- ctx.Err()
- return
- default:
- batchSize := f.batchSize
- if batchSize <= 0 {
- batchSize = 10
- }
-
- keys := make([]int64, batchSize)
- data := make([][]byte, batchSize)
- partIDs := make([]uint64, batchSize)
-
- for i := 0; i < batchSize; i++ {
- keys[i] = key
- prefix := f.traceIDPrefix
- if prefix == "" {
- prefix = "trace"
- }
- traceID := prefix + "-" + strconv.Itoa(counter)
- data[i] = encodeTraceIDForTest(traceID)
- partIDs[i] = uint64(1000 + counter)
- key++
- counter++
- }
-
- resp := &sidx.QueryResponse{
- Keys: keys,
- Data: data,
- PartIDs: partIDs,
- }
-
- select {
- case <-ctx.Done():
- errCh <- ctx.Err()
- return
- case results <- resp:
- }
- }
- }
- }()
-
- return results, errCh
-}
-
-func (f *fakeSIDXInfinite) IntroduceMemPart(uint64, *sidx.MemPart) { panic("not implemented") }
-func (f *fakeSIDXInfinite) IntroduceFlushed(*sidx.FlusherIntroduction) {}
-func (f *fakeSIDXInfinite) IntroduceMerged(*sidx.MergerIntroduction) func() { return func() {} }
-func (f *fakeSIDXInfinite) ConvertToMemPart([]sidx.WriteRequest, int64, *int64, *int64) (*sidx.MemPart, error) {
- panic("not implemented")
-}
-
-func (f *fakeSIDXInfinite) Query(context.Context, sidx.QueryRequest) (*sidx.QueryResponse, error) {
- panic("not implemented")
-}
-func (f *fakeSIDXInfinite) Stats(context.Context) (*sidx.Stats, error) { return &sidx.Stats{}, nil }
-func (f *fakeSIDXInfinite) Close() error { return nil }
-func (f *fakeSIDXInfinite) Flush(map[uint64]struct{}) (*sidx.FlusherIntroduction, error) {
- panic("not implemented")
-}
-
-func (f *fakeSIDXInfinite) Merge(<-chan struct{}, map[uint64]struct{}, uint64, func([]byte) bool) (*sidx.MergerIntroduction, error) {
- panic("not implemented")
-}
-
-func (f *fakeSIDXInfinite) StreamingParts(map[uint64]struct{}, string, uint32, string) ([]queue.StreamingPartData, []func()) {
- panic("not implemented")
-}
-
-func (f *fakeSIDXInfinite) PartPaths(map[uint64]struct{}) map[uint64]string {
- return map[uint64]string{}
-}
-func (f *fakeSIDXInfinite) IntroduceSynced(map[uint64]struct{}) func() { return func() {} }
-func (f *fakeSIDXInfinite) TakeFileSnapshot(_ string) error { return nil }
-func (f *fakeSIDXInfinite) PrepareMemPart(uint64, *sidx.MemPart) func(cur *sidx.Snapshot) *sidx.Snapshot {
- return func(cur *sidx.Snapshot) *sidx.Snapshot { return cur }
-}
-
-func (f *fakeSIDXInfinite) PrepareFilePart(uint64, string) func(cur *sidx.Snapshot) *sidx.Snapshot {
- return func(cur *sidx.Snapshot) *sidx.Snapshot { return cur }
-}
-
-func (f *fakeSIDXInfinite) PrepareFlushed(*sidx.FlusherIntroduction) func(cur *sidx.Snapshot) *sidx.Snapshot {
- return func(cur *sidx.Snapshot) *sidx.Snapshot { return cur }
-}
-
-func (f *fakeSIDXInfinite) PrepareMerged(*sidx.MergerIntroduction) func(cur *sidx.Snapshot) *sidx.Snapshot {
- return func(cur *sidx.Snapshot) *sidx.Snapshot { return cur }
-}
-
-func (f *fakeSIDXInfinite) PrepareSynced(map[uint64]struct{}) func(cur *sidx.Snapshot) *sidx.Snapshot {
- return func(cur *sidx.Snapshot) *sidx.Snapshot { return cur }
-}
-func (f *fakeSIDXInfinite) CurrentSnapshot() *sidx.Snapshot { return nil }
-func (f *fakeSIDXInfinite) ReplaceSnapshot(*sidx.Snapshot) {}
-
-// TestStreamSIDXTraceBatches_InfiniteChannelContinuesUntilCanceled verifies that
-// the streaming pipeline continues streaming from an infinite channel until context is canceled.
-func TestStreamSIDXTraceBatches_InfiniteChannelContinuesUntilCanceled(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 5,
- }
-
- sidxInstance := &fakeSIDXInfinite{
- batchSize: 10,
- keyStart: 1,
- traceIDPrefix: "inf",
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0 /* no maxTraceSize limit */)
-
- const targetTraceIDs = 50
- totalTraceIDs := 0
- seenIDs := make(map[string]struct{})
-
- for batch := range batchCh {
- if batch.err != nil {
- // Context cancellation is expected
- if errors.Is(batch.err, context.Canceled) {
- break
- }
- t.Fatalf("unexpected error batch: %v", batch.err)
- }
-
- for _, ids := range batch.traceIDs {
- for _, tid := range ids {
- if _, exists := seenIDs[tid]; exists {
- t.Fatalf("duplicate trace ID: %s", tid)
- }
- seenIDs[tid] = struct{}{}
- totalTraceIDs++
- }
- }
-
- // Cancel after we've received enough traces to prove it's streaming
- if totalTraceIDs >= targetTraceIDs {
- cancel()
- }
- }
-
- // Should have received at least targetTraceIDs results
- if totalTraceIDs < targetTraceIDs {
- t.Fatalf("expected at least %d trace IDs, got %d", targetTraceIDs, totalTraceIDs)
- }
-
- t.Logf("Successfully received %d unique trace IDs from infinite stream before cancellation", totalTraceIDs)
-}
-
-// TestStreamSIDXTraceBatches_InfiniteChannelCancellation verifies that
-// canceling the context properly stops all goroutines in the streaming pipeline
-// and cancels the SIDX context to close the infinite channel.
-func TestStreamSIDXTraceBatches_InfiniteChannelCancellation(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 5,
- }
-
- sidxInstance := &fakeSIDXInfinite{
- batchSize: 10,
- keyStart: 1,
- traceIDPrefix: "cancel",
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0 /* no maxTraceSize limit */)
-
- // Read a few batches, then cancel
- batchesRead := 0
- const batchesToRead = 3
-
- for batch := range batchCh {
- if batch.err != nil {
- // Context cancellation error is expected
- if errors.Is(batch.err, context.Canceled) {
- t.Logf("Received expected cancellation error: %v", batch.err)
- break
- }
- t.Fatalf("unexpected error: %v", batch.err)
- }
-
- batchesRead++
- if batchesRead >= batchesToRead {
- // Cancel the context to stop the infinite stream
- cancel()
- }
- }
-
- // Verify we read some batches before cancellation
- if batchesRead < batchesToRead {
- t.Fatalf("expected to read at least %d batches, got %d", batchesToRead, batchesRead)
- }
-
- t.Logf("Successfully canceled infinite stream after reading %d batches", batchesRead)
-}
-
-// TestStreamSIDXTraceBatches_InfiniteChannelGoroutineCleanup verifies that
-// all goroutines are properly cleaned up when context is canceled.
-func TestStreamSIDXTraceBatches_InfiniteChannelGoroutineCleanup(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 5,
- }
-
- sidxInstance := &fakeSIDXInfinite{
- batchSize: 5,
- keyStart: 1,
- traceIDPrefix: "cleanup",
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{sidxInstance}, req, 0 /* no limit */)
-
- totalTraceIDs := 0
- batchesRead := 0
-
- for batch := range batchCh {
- if batch.err != nil {
- // Context cancellation is expected
- if errors.Is(batch.err, context.Canceled) {
- t.Logf("Received expected cancellation error")
- break
- }
- t.Fatalf("unexpected error: %v", batch.err)
- }
-
- for _, ids := range batch.traceIDs {
- totalTraceIDs += len(ids)
- }
- batchesRead++
-
- // Cancel after reading a few batches
- if batchesRead >= 3 {
- cancel()
- }
- }
-
- // Channel should be closed
- _, ok := <-batchCh
- if ok {
- t.Fatal("channel should be closed")
- }
-
- if totalTraceIDs == 0 {
- t.Fatal("expected some trace IDs before cancellation")
- }
-
- t.Logf("Successfully cleaned up: read %d batches, %d trace IDs", batchesRead, totalTraceIDs)
-}
-
-// TestStreamSIDXTraceBatches_MultipleInfiniteSIDX verifies that
-// the streaming pipeline can handle multiple infinite SIDX instances
-// and properly merge their results until context is canceled.
-func TestStreamSIDXTraceBatches_MultipleInfiniteSIDX(t *testing.T) {
- req := sidx.QueryRequest{
- Order: &index.OrderBy{Sort: modelv1.Sort_SORT_ASC},
- MaxBatchSize: 10,
- }
-
- const targetTraceIDs = 50
-
- // Create multiple infinite SIDX instances with different key ranges
- sidxInstances := []sidx.SIDX{
- &fakeSIDXInfinite{
- batchSize: 5,
- keyStart: 1, // Keys: 1, 2, 3, ...
- traceIDPrefix: "s1",
- },
- &fakeSIDXInfinite{
- batchSize: 5,
- keyStart: 1000, // Keys: 1000, 1001, 1002, ...
- traceIDPrefix: "s2",
- },
- &fakeSIDXInfinite{
- batchSize: 5,
- keyStart: 2000, // Keys: 2000, 2001, 2002, ...
- traceIDPrefix: "s3",
- },
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- var tr trace
- batchCh, _ := tr.streamSIDXTraceBatches(ctx, sidxInstances, req, 0 /* no limit */)
-
- totalTraceIDs := 0
- seenIDs := make(map[string]struct{})
- var keys []int64
-
- for batch := range batchCh {
- if batch.err != nil {
- // Context cancellation is expected
- if errors.Is(batch.err, context.Canceled) {
- break
- }
- t.Fatalf("unexpected error batch: %v", batch.err)
- }
-
- // Use traceIDsOrder to maintain the correct order from SIDX stream
- for _, tid := range batch.traceIDsOrder {
- if _, exists := seenIDs[tid]; exists {
- t.Fatalf("duplicate trace ID: %s", tid)
- }
- seenIDs[tid] = struct{}{}
- keys = append(keys, batch.keys[tid])
- totalTraceIDs++
- }
-
- // Cancel after we've received enough to verify the merge is working
- if totalTraceIDs >= targetTraceIDs {
- cancel()
- }
- }
-
- if totalTraceIDs < targetTraceIDs {
- t.Fatalf("expected at least %d trace IDs, got %d", targetTraceIDs, totalTraceIDs)
- }
-
- // Verify keys are in ascending order (due to heap merge)
- for i := 1; i < len(keys); i++ {
- if keys[i] < keys[i-1] {
- t.Fatalf("keys not in ascending order: keys[%d]=%d, keys[%d]=%d", i-1, keys[i-1], i, keys[i])
- }
- }
-
- t.Logf("Successfully merged %d trace IDs from %d infinite SIDX instances", totalTraceIDs, len(sidxInstances))
-}
diff --git a/banyand/trace/svc_liaison.go b/banyand/trace/svc_liaison.go
index f4612cd76..5eb86ace8 100644
--- a/banyand/trace/svc_liaison.go
+++ b/banyand/trace/svc_liaison.go
@@ -143,7 +143,7 @@ func (l *liaison) FlagSet() *run.FlagSet {
"percentage of BanyanDB's allowed disk usage allocated to failed parts storage. "+
"Calculated as: totalDisk * trace-max-disk-usage-percent * failed-parts-max-size-percent / 10000. "+
"Set to 0 to disable copying failed parts. Valid range: 0-100")
- bindVectorizedFlags(fs, &l.option.vectorized)
+ bindVectorizedFlags(fs, &l.option.vectorized, &l.option.vectorizedEnabled)
return fs
}
@@ -165,6 +165,9 @@ func (l *liaison) Validate() error {
if l.failedPartsMaxSizePercent < 0 || l.failedPartsMaxSizePercent > 100 {
return fmt.Errorf("invalid failed-parts-max-size-percent: %d%%. Must be between 0 and 100", l.failedPartsMaxSizePercent)
}
+ if !l.option.vectorizedEnabled {
+ return errRowQueryRemoved
+ }
return l.option.vectorized.Validate()
}
@@ -178,8 +181,7 @@ func (l *liaison) Role() databasev1.Role {
func (l *liaison) PreRun(ctx context.Context) error {
l.l = logger.GetLogger(l.Name())
- data.SetTraceWireModeRaw(l.option.vectorized.Enabled)
- l.l.Info().Bool("trace_wire_mode_raw", l.option.vectorized.Enabled).Msg("trace wire mode published (liaison)")
+ data.SetTraceWireModeRaw(true)
l.l.Info().Msg("memory protector is initialized in PreRun")
l.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(l.l, l.pm.GetLimit())
var err error
diff --git a/banyand/trace/svc_standalone.go b/banyand/trace/svc_standalone.go
index 751e56b5b..3c5945d8d 100644
--- a/banyand/trace/svc_standalone.go
+++ b/banyand/trace/svc_standalone.go
@@ -105,7 +105,7 @@ func (s *standalone) FlagSet() *run.FlagSet {
s.option.mergePolicy = newDefaultMergePolicy()
fs.IntVar(&s.option.mergePolicy.maxParts, "trace-max-merge-parts", s.option.mergePolicy.maxParts, "the maximum number of parts to merge at once")
fs.VarP(&s.option.mergePolicy.maxFanOutSize, "trace-max-fan-out-size", "", "the upper bound of a single file size after merge of trace")
- bindVectorizedFlags(fs, &s.option.vectorized)
+ bindVectorizedFlags(fs, &s.option.vectorized, &s.option.vectorizedEnabled)
s.option.mergeGraceDefault = defaultTracePipelineMergeGrace
s.option.finalizeGraceDefault = defaultTracePipelineFinalizeGrace
fs.BoolVar(&s.option.nativePipelineEnabled, "trace-pipeline-native-plugin-enabled", false, "enable the native plugin pipeline for in-merge trace retention")
@@ -137,6 +137,9 @@ func (s *standalone) Validate() error {
if s.retentionConfig.Cooldown <= 0 {
return errors.New("trace-retention-cooldown must be greater than 0")
}
+ if !s.option.vectorizedEnabled {
+ return errRowQueryRemoved
+ }
return s.option.vectorized.Validate()
}
@@ -150,10 +153,8 @@ func (s *standalone) Role() databasev1.Role {
func (s *standalone) PreRun(ctx context.Context) error {
s.l = logger.GetLogger(s.Name())
- // Native columnar wire frame for the data↔liaison query hop follows the
- // vectorized flag (mirrors measure's wire-mode wiring).
- data.SetTraceWireModeRaw(s.option.vectorized.Enabled)
- s.l.Info().Bool("trace_wire_mode_raw", s.option.vectorized.Enabled).Msg("trace wire mode published (standalone)")
+ // The data↔liaison query hop always carries the native columnar wire frame.
+ data.SetTraceWireModeRaw(true)
s.l.Info().Msg("memory protector is initialized in PreRun")
s.lfs = fs.NewLocalFileSystemWithLoggerAndLimit(s.l, s.pm.GetLimit())
var err error
diff --git a/banyand/trace/trace.go b/banyand/trace/trace.go
index b90e09a39..8e6968b1a 100644
--- a/banyand/trace/trace.go
+++ b/banyand/trace/trace.go
@@ -23,6 +23,7 @@ package trace
import (
"context"
+ "errors"
"sync/atomic"
"time"
@@ -67,6 +68,7 @@ type option struct {
vectorized vtrace.VectorizedConfig
decideTimeoutCircuitBreak int
nativePipelineEnabled bool
+ vectorizedEnabled bool
benchmarkMergeBlocked bool
isHot bool
}
@@ -167,11 +169,14 @@ func (t *trace) VectorizedConfig() vtrace.VectorizedConfig {
return t.vectorized
}
+// errRowQueryRemoved reports a startup attempt to fall back to the deleted row-based trace query path.
+var errRowQueryRemoved = errors.New("--trace-vectorized-enabled=false: row-based query was removed in 0.12.0, see apache/skywalking#13998")
+
// bindVectorizedFlags wires VectorizedConfig fields to a run.FlagSet.
-func bindVectorizedFlags(flagS *run.FlagSet, cfg *vtrace.VectorizedConfig) {
+func bindVectorizedFlags(flagS *run.FlagSet, cfg *vtrace.VectorizedConfig, vectorizedEnabled *bool) {
defaults := vtrace.DefaultConfig()
- flagS.BoolVar(&cfg.Enabled, "trace-vectorized-enabled", defaults.Enabled,
- "enable the vectorized trace query path")
+ flagS.BoolVar(vectorizedEnabled, "trace-vectorized-enabled", true,
+ "deprecated no-op, removed in 0.13.0; =false aborts startup because the row-based trace query path no longer exists")
flagS.IntVar(&cfg.BatchSize, "trace-vectorized-batch-size", defaults.BatchSize,
"row count per vectorized trace batch")
flagS.IntVar(&cfg.QueryMemoryMiB, "trace-vectorized-query-memory-mib", defaults.QueryMemoryMiB,
diff --git a/banyand/trace/vectorized_bench_test.go b/banyand/trace/vectorized_bench_test.go
deleted file mode 100644
index 135cd27ce..000000000
--- a/banyand/trace/vectorized_bench_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "context"
- "testing"
-
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/pkg/query/model"
-)
-
-// BenchmarkRowBasedVsVectorized_Lookup compares the original row-based goroutine path
-// (startBlockScanStage → queryResult) against the new vectorized pull path
-// (buildVectorizedScanBatch → vectorizedTraceQueryResult) for traceID lookup queries.
-// Run with: go test -bench=BenchmarkRowBasedVsVectorized_Lookup -benchmem ./banyand/trace/.
-func BenchmarkRowBasedVsVectorized_Lookup(b *testing.B) {
- tst, cleanup := newParityTSTable(b, tsTS1, tsTS2)
- defer cleanup()
-
- tr := newParityTrace()
- tables := []*tsTable{tst}
- qo := queryOptions{
- TraceQueryOptions: model.TraceQueryOptions{TagProjection: allTagProjections},
- schemaTagTypes: testSchemaTagTypes,
- }
- traceIDs := []string{"trace1", "trace2", "trace3"}
-
- b.Run("RowBased", func(b *testing.B) {
- b.ResetTimer()
- for benchIdx := 0; benchIdx < b.N; benchIdx++ {
- ctx := context.Background()
- pushRes := pushLookupResult(ctx, tr, tables, qo, traceIDs, 0)
- for {
- r := pushRes.Pull()
- if r == nil {
- break
- }
- }
- b.StopTimer()
- pushRes.Release()
- b.StartTimer()
- }
- })
-
- b.Run("Vectorized", func(b *testing.B) {
- b.ResetTimer()
- for benchIdx := 0; benchIdx < b.N; benchIdx++ {
- ctx := context.Background()
- pullRes, pullErr := pullLookupResult(ctx, tr, tables, qo, traceIDs, 0)
- require.NoError(b, pullErr)
- for {
- r := pullRes.Pull()
- if r == nil {
- break
- }
- }
- b.StopTimer()
- pullRes.Release()
- b.StartTimer()
- }
- })
-}
diff --git a/banyand/trace/vectorized_flag_test.go b/banyand/trace/vectorized_flag_test.go
new file mode 100644
index 000000000..4d51696d5
--- /dev/null
+++ b/banyand/trace/vectorized_flag_test.go
@@ -0,0 +1,41 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+ vtrace "github.com/apache/skywalking-banyandb/pkg/query/vectorized/trace"
+)
+
+func TestVectorizedEnabledFalseAbortsStartup(t *testing.T) {
+ newOption := func(vectorizedEnabled bool) option {
+ return option{vectorized: vtrace.DefaultConfig(), vectorizedEnabled: vectorizedEnabled}
+ }
+ retention := storage.RetentionConfig{HighWatermark: 95, LowWatermark: 85, Cooldown: time.Second}
+
+ require.ErrorIs(t, (&standalone{root: "/tmp", retentionConfig: retention, option: newOption(false)}).Validate(), errRowQueryRemoved)
+ require.NoError(t, (&standalone{root: "/tmp", retentionConfig: retention, option: newOption(true)}).Validate())
+
+ require.ErrorIs(t, (&liaison{root: "/tmp", option: newOption(false)}).Validate(), errRowQueryRemoved)
+ require.NoError(t, (&liaison{root: "/tmp", option: newOption(true)}).Validate())
+}
diff --git a/banyand/trace/vectorized_parity_test.go b/banyand/trace/vectorized_parity_test.go
index 6432c4043..759c42b75 100644
--- a/banyand/trace/vectorized_parity_test.go
+++ b/banyand/trace/vectorized_parity_test.go
@@ -23,9 +23,7 @@ import (
"testing"
"time"
- "github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
- "google.golang.org/protobuf/testing/protocmp"
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
@@ -57,6 +55,15 @@ func collectResults(t *testing.T, result model.TraceQueryResult) []*model.TraceR
return got
}
+// traceIDsOf projects the trace IDs out of a result slice, in result order.
+func traceIDsOf(results []*model.TraceResult) []string {
+ ids := make([]string, 0, len(results))
+ for _, r := range results {
+ ids = append(ids, r.TID)
+ }
+ return ids
+}
+
// newParityTSTable sets up an in-memory tsTable populated with the given trace sets.
func newParityTSTable(tb testing.TB, traceSets ...*traces) (*tsTable, func()) {
tb.Helper()
@@ -79,26 +86,7 @@ func newParityTSTable(tb testing.TB, traceSets ...*traces) (*tsTable, func()) {
func newParityTrace() *trace {
return &trace{
pm: protector.Nop{},
- vectorized: vtrace.VectorizedConfig{Enabled: true, BatchSize: 10, QueryMemoryMiB: 100},
- }
-}
-
-func pushLookupResult(
- ctx context.Context,
- tr *trace,
- tables []*tsTable,
- qo queryOptions,
- traceIDs []string,
- maxTraceSize int,
-) *queryResult {
- pushKeys := make(map[string]int64)
- traceBatchCh := staticTraceBatchSource(ctx, traceIDs, maxTraceSize, pushKeys)
- cursorBatchCh := tr.startBlockScanStage(ctx, tables, qo, traceBatchCh)
- return &queryResult{
- ctx: ctx,
- keys: pushKeys,
- tagProjection: qo.TagProjection,
- cursorBatchCh: cursorBatchCh,
+ vectorized: vtrace.VectorizedConfig{BatchSize: 10, QueryMemoryMiB: 100},
}
}
@@ -122,26 +110,6 @@ func pullLookupResult(
return newVectorizedTraceQueryResult(ctx, sb, qo, nil, nil, nil, nil)
}
-func pushOrderResult(
- ctx context.Context,
- tr *trace,
- tables []*tsTable,
- qo queryOptions,
- fakeSIDXInst *fakeSIDXWithSync,
- req sidx.QueryRequest,
- maxTraceSize int,
-) *queryResult {
- traceBatchCh, streamDone := tr.streamSIDXTraceBatches(ctx, []sidx.SIDX{fakeSIDXInst}, req, maxTraceSize)
- cursorBatchCh := tr.startBlockScanStage(ctx, tables, qo, traceBatchCh)
- return &queryResult{
- ctx: ctx,
- keys: make(map[string]int64),
- tagProjection: qo.TagProjection,
- cursorBatchCh: cursorBatchCh,
- streamDone: streamDone,
- }
-}
-
func pullOrderResult(
ctx context.Context,
tr *trace,
@@ -162,9 +130,11 @@ func pullOrderResult(
return newVectorizedTraceQueryResult(ctx, sb, qo, nil, nil, nil, nil)
}
-// TestVectorizedParityLookup verifies that the push and pull paths return
-// byte-identical model.TraceResult sequences for traceID lookup mode.
-func TestVectorizedParityLookup(t *testing.T) {
+// TestVectorizedLookup pins the vectorized traceID-lookup result: the traces the
+// requested IDs resolve to, the MaxTraceSize cap, and the AC8 contract that an
+// identity-only projection (nil or empty after omitIdentityTagProjection) yields
+// r.Tags == nil.
+func TestVectorizedLookup(t *testing.T) {
tst, cleanup := newParityTSTable(t, tsTS1, tsTS2)
defer cleanup()
@@ -197,37 +167,27 @@ func TestVectorizedParityLookup(t *testing.T) {
schemaTagTypes: testSchemaTagTypes,
}
- pushRes := pushLookupResult(ctx, tr, tables, qo, tt.traceIDs, tt.maxTraceSize)
- defer pushRes.Release()
- pushGot := collectResults(t, pushRes)
-
pullRes, pullErr := pullLookupResult(ctx, tr, tables, qo, tt.traceIDs, tt.maxTraceSize)
require.NoError(t, pullErr)
defer pullRes.Release()
pullGot := collectResults(t, pullRes)
- require.Len(t, pushGot, tt.wantCount, "push path result count")
- require.Len(t, pullGot, tt.wantCount, "pull path result count")
+ require.Len(t, pullGot, tt.wantCount, "result count")
+ require.Subset(t, tt.traceIDs, traceIDsOf(pullGot), "resolved a trace that was not requested")
if tt.wantNilTags {
- for _, r := range pushGot {
- require.Nil(t, r.Tags, "push path must produce nil Tags for empty projection")
- }
for _, r := range pullGot {
- require.Nil(t, r.Tags, "pull path must produce nil Tags for empty projection")
+ require.Nil(t, r.Tags, "must produce nil Tags for empty projection")
}
}
-
- if diff := cmp.Diff(pushGot, pullGot, protocmp.Transform()); diff != "" {
- t.Errorf("push vs pull lookup mismatch (-push +pull):\n%s", diff)
- }
})
}
}
-// TestVectorizedParityOrderMode verifies that the push and pull paths return
-// byte-identical model.TraceResult sequences for order mode (ASC and DESC).
-func TestVectorizedParityOrderMode(t *testing.T) {
+// TestVectorizedOrderMode pins the vectorized order-mode result: the traces come
+// back in the SIDX key order (ASC and DESC), and MaxTraceSize caps the
+// materialized set to the sorted top-N rather than truncating arbitrarily.
+func TestVectorizedOrderMode(t *testing.T) {
tst, cleanup := newParityTSTable(t, tsTS1)
defer cleanup()
@@ -300,8 +260,6 @@ func TestVectorizedParityOrderMode(t *testing.T) {
sids := make([]common.SeriesID, len(tt.traceIDs))
partIDs := make([]uint64, len(tt.traceIDs))
- // Push (StreamingQuery) and pull (QuerySync) share one response slice so
- // they always use identical data and cannot drift if either path is rewired.
sharedResp := []*sidx.QueryResponse{
{Keys: tt.keys, Data: respData, SIDs: sids, PartIDs: partIDs},
}
@@ -310,28 +268,20 @@ func TestVectorizedParityOrderMode(t *testing.T) {
syncResponses: sharedResp,
}
- pushRes := pushOrderResult(ctx, tr, tables, qo, fakeSIDXInst, req, tt.maxTraceSize)
- defer pushRes.Release()
- pushGot := collectResults(t, pushRes)
-
pullRes, pullErr := pullOrderResult(ctx, tr, tables, qo, fakeSIDXInst, req, tt.maxTraceSize)
require.NoError(t, pullErr)
defer pullRes.Release()
pullGot := collectResults(t, pullRes)
- // The pull path caps the materialized set at MaxTraceSize; the push path streams
- // all matches (the outer traceLimit caps it in production). With no cap the two
- // are identical; with a cap the pull result is the sorted MaxTraceSize-prefix.
+ // MaxTraceSize (= offset+limit) caps the materialized trace set to the sorted
+ // top-N; tt.traceIDs is already listed in the sort direction under test, so the
+ // expectation is its MaxTraceSize-prefix.
wantPull := tt.wantCount
if tt.wantPullCount > 0 {
wantPull = tt.wantPullCount
}
- require.Len(t, pushGot, tt.wantCount, "push path result count")
- require.Len(t, pullGot, wantPull, "pull path result count")
-
- if diff := cmp.Diff(pushGot[:wantPull], pullGot, protocmp.Transform()); diff != "" {
- t.Errorf("pull path must equal the sorted MaxTraceSize-prefix of push (-push +pull):\n%s", diff)
- }
+ require.Len(t, pullGot, wantPull, "result count")
+ require.Equal(t, tt.traceIDs[:wantPull], traceIDsOf(pullGot), "traces not returned in sort-key order")
})
}
}
@@ -401,7 +351,7 @@ func TestVectorizedMemoryQuotaAC5(t *testing.T) {
// Pull path with unlimited quota succeeds.
unlimitedTrace := &trace{
pm: protector.Nop{},
- vectorized: vtrace.VectorizedConfig{Enabled: true, BatchSize: 10, QueryMemoryMiB: 100},
+ vectorized: vtrace.VectorizedConfig{BatchSize: 10, QueryMemoryMiB: 100},
}
qo.traceIDs = []string{"trace1"}
unlimitedBatch, unlimitedBatchErr := unlimitedTrace.buildVectorizedPhase1TraceBatch(ctx, qo, nil, sidx.QueryRequest{}, false, 0)
@@ -415,7 +365,7 @@ func TestVectorizedMemoryQuotaAC5(t *testing.T) {
// have been accumulated yet, matching push path's "quota exceeded" early-exit.
zeroQuotaTrace := &trace{
pm: zeroQuotaProtector{},
- vectorized: vtrace.VectorizedConfig{Enabled: true, BatchSize: 10, QueryMemoryMiB: 100},
+ vectorized: vtrace.VectorizedConfig{BatchSize: 10, QueryMemoryMiB: 100},
}
qo.traceIDs = []string{"trace1"}
zeroBatch, zeroBatchErr := zeroQuotaTrace.buildVectorizedPhase1TraceBatch(ctx, qo, nil, sidx.QueryRequest{}, false, 0)
@@ -466,10 +416,10 @@ func TestVectorizedTracingSpansAC7(t *testing.T) {
require.Contains(t, allMsgs, "scan-blocks", "AC7: pull path must emit scan-blocks span")
}
-// TestVectorizedTTLExpiredParity verifies AC9: when TTL-expired segment filtering
-// produces an empty tables slice (as SelectSegments returns nothing for expired data),
-// both push and pull paths return zero results identically.
-func TestVectorizedTTLExpiredParity(t *testing.T) {
+// TestVectorizedTTLExpired verifies AC9: when TTL-expired segment filtering
+// produces an empty tables slice (as SelectSegments returns nothing for expired
+// data), the vectorized path returns zero results rather than erroring.
+func TestVectorizedTTLExpired(t *testing.T) {
tr := newParityTrace()
ctx := context.Background()
qo := queryOptions{
@@ -481,15 +431,9 @@ func TestVectorizedTTLExpiredParity(t *testing.T) {
// Empty tables simulates all segments being TTL-expired (SelectSegments returned nothing).
emptyTables := []*tsTable{}
- pushRes := pushLookupResult(ctx, tr, emptyTables, qo, traceIDs, 0)
- defer pushRes.Release()
- pushGot := collectResults(t, pushRes)
-
pullRes, pullErr := pullLookupResult(ctx, tr, emptyTables, qo, traceIDs, 0)
require.NoError(t, pullErr)
defer pullRes.Release()
- pullGot := collectResults(t, pullRes)
- require.Empty(t, pushGot, "push path must return no results for TTL-expired (empty) tables")
- require.Empty(t, pullGot, "pull path must return no results for TTL-expired (empty) tables")
+ require.Empty(t, collectResults(t, pullRes), "must return no results for TTL-expired (empty) tables")
}
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 0322eb585..7c939fa5b 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -1106,7 +1106,7 @@ Contains shard information for proper deduplication.
| ----- | ---- | ----- | ----------- |
| data_points | [InternalDataPoint](#banyandb-measure-v1-InternalDataPoint) | repeated | data_points with shard information |
| trace | [banyandb.common.v1.Trace](#banyandb-common-v1-Trace) | | trace contains the trace information of the query when trace is enabled |
-| raw_frame_body | [bytes](#bytes) | | raw_frame_body contains the vectorized raw frame body when raw wire mode is enabled |
+| raw_frame_body | [bytes](#bytes) | | raw_frame_body contains the vectorized raw frame body |
diff --git a/docs/operation/observability/tracing.md b/docs/operation/observability/tracing.md
index caf48362f..818ff67b9 100644
--- a/docs/operation/observability/tracing.md
+++ b/docs/operation/observability/tracing.md
@@ -22,7 +22,7 @@ The result will include the tracing data in the response. The duration time unit
## Vectorized Measure Query Tracing
-Vectorized distributed measure queries use the existing `pkg/query.Tracer` and `common.v1.Trace` tree. Tracing is opt-in through `QueryRequest.trace`; trace-off raw wire responses remain raw frame bytes.
+Vectorized distributed measure queries use the existing `pkg/query.Tracer` and `common.v1.Trace` tree. Tracing is opt-in through `QueryRequest.trace`; trace-off responses remain raw frame bytes.
### Span shape
@@ -324,4 +324,4 @@ The summary fully represents every frame's decode duration. `decode_ns_p99 ≫ p
### Wire contract
-Under raw wire mode, trace-off responses are still opaque raw frame bytes beginning with `RawFrameMagicLeadingByte`. Trace-on responses use the existing `measure.v1.InternalQueryResponse` envelope with `raw_frame_body` and `trace` populated. Proto `data_points` responses under raw mode are rejected loudly by the vectorized collector.
+Trace-off responses are opaque raw frame bytes beginning with `RawFrameMagicLeadingByte`. Trace-on responses use the existing `measure.v1.InternalQueryResponse` envelope with `raw_frame_body` and `trace` populated. Proto `data_points` responses are rejected loudly by the vectorized collector.
diff --git a/docs/operation/troubleshooting/error-checklist.md b/docs/operation/troubleshooting/error-checklist.md
index a848483ea..5756adfbd 100644
--- a/docs/operation/troubleshooting/error-checklist.md
+++ b/docs/operation/troubleshooting/error-checklist.md
@@ -46,4 +46,3 @@ Follow this procedure to address the identified error type:
- [Troubleshooting Query Issues](./query.md)
- [Troubleshooting Installation Issues](./install.md)
- [Common Issues](./common-issues.md)
-- [Measure Vec Flag-Off Rollback](./measure-vec-flag-off-rollback.md)
diff --git a/docs/operation/troubleshooting/measure-vec-flag-off-rollback.md b/docs/operation/troubleshooting/measure-vec-flag-off-rollback.md
deleted file mode 100644
index 408922f04..000000000
--- a/docs/operation/troubleshooting/measure-vec-flag-off-rollback.md
+++ /dev/null
@@ -1,71 +0,0 @@
-# Measure Vec Flag-Off Rollback
-
-This runbook documents the kill-switch for the vectorized distributed measure query path and explains how to roll back to the legacy row-path if a production incident requires it.
-
-## Background
-
-Since Phase 6 of the vectorized distributed plan (see `banyand/dquery/measure.go`), the routing decision in the distributed measure query handler is a single boolean:
-
-```
-data.MeasureWireModeRaw() == true → vecplan.AnalyzeDistributed (production default)
-data.MeasureWireModeRaw() == false → logical_measure.DistributedAnalyze (rollback path)
-```
-
-There is no longer a `useVecDistributedMeasurePlan` predicate or a per-shape gate. Every request shape (non-agg, Agg, OrderBy-by-index-rule, multi-group, Top, GroupBy, and combinations) goes through the vec plan when the flag is on.
-
-## What the flag controls
-
-`data.MeasureWireModeRaw()` reads a per-process atomic boolean set at startup by `data.SetMeasureWireModeRaw(true)`. When true, the data-node wire codec switches from `proto.Marshal`/`proto.Unmarshal` (`InternalQueryResponse`) to Apache Arrow record-batch frames. The liaison node's distributed handler then dispatches to `vecplan.AnalyzeDistributed` instead of the row-path `distributedPlan`.
-
-## How to flip it off in a running cluster
-
-The flag is set at process startup. To disable it:
-
-1. Locate the BanyanDB node configuration (typically `bydb.yaml` or the Helm values file).
-2. Set the measure wire mode to proto (row) mode. The exact configuration key is `measure.wireMode: proto` (or the equivalent environment variable `BYDB_MEASURE_WIRE_MODE=proto`).
-3. Perform a rolling restart of liaison and data nodes. Because the flag controls per-process codec selection, both liaison and data nodes must run the same mode; a mixed-mode cluster will return decode errors.
-
-After restart, verify with:
-
-```bash
-bydbctl measure query --name <name> --group <group> --limit 1
-```
-
-A successful response confirms the row path is active.
-
-## Which code path takes over when off
-
-When `data.MeasureWireModeRaw()` returns false, the liaison's `Rev` handler in `banyand/dquery/measure.go` calls:
-
-```go
-logical_measure.DistributedAnalyze(queryCriteria, schemas)
-```
-
-This returns a `distributedPlan` (in `pkg/query/logical/measure/measure_plan_distributed.go`) that:
-
-- Broadcasts `InternalQueryRequest` to data nodes using `proto.Marshal` wire encoding.
-- Receives `*measurev1.InternalQueryResponse` from each data node.
-- Performs a k-way sort-merge via `sortedMIterator` using `sort.NewItemIter`.
-- Deduplicates by `(Sid, Timestamp)` hash and version across replicas.
-
-The `[]byte` raw-frame branch that previously existed in this code path was removed in Phase 6 — it was unreachable once the liaison always routes raw-mode requests through vec.
-
-## Operational constraints of the flag-off path
-
-- **Higher latency**: each data node serialises results to proto, sends over the wire, and the liaison deserialises before merge. Under the vec path, data stays in columnar Arrow format across the wire and into the merge.
-- **No native multi-group merge**: the row-path `sortedMIterator` merges across data nodes but does not perform cross-group native columnar aggregation. Multi-group queries still work but via the proto round-trip.
-- **No native Top-N merge**: Top-N is evaluated per data node; the liaison re-sorts and truncates. Under vec, Top-N is evaluated natively in the columnar pipeline.
-- **Proto size overhead**: `InternalQueryResponse` encodes every data point with its full tag and field set in proto format, which is larger than Arrow record batches for wide schemas.
-
-## When to use the rollback
-
-Use the flag-off path only during incident response when the vec path produces incorrect results or panics that cannot be fixed with a hot patch. The vec path is the production default and has been validated by the full distributed integration suite. After the incident is resolved, re-enable raw wire mode and perform another rolling restart.
-
-## Relevant source locations
-
-| File | Purpose |
-|---|---|
-| `banyand/dquery/measure.go` | Liaison routing — branches on `data.MeasureWireModeRaw()` |
-| `api/data/codec.go` | `SetMeasureWireModeRaw` / `MeasureWireModeRaw` definitions |
-| `pkg/query/logical/measure/measure_plan_distributed.go` | Row-path distributed plan (`distributedPlan`) |
-| `pkg/query/vectorized/measure/plan/` | Vec distributed plan (`vecplan.AnalyzeDistributed`) |
diff --git a/docs/operation/troubleshooting/query.md b/docs/operation/troubleshooting/query.md
index d88078835..f25f1dc7b 100644
--- a/docs/operation/troubleshooting/query.md
+++ b/docs/operation/troubleshooting/query.md
@@ -66,7 +66,7 @@ Please refer to the [Troubleshooting No Data Issue](./no-data.md) guide to ident
Use query tracing to understand execution plans and identify bottlenecks. To enable query tracing, set the `trace` field to `true` in the [MeasureQueryRequest](../../api-reference.md#queryrequest) and [StreamQueryRequest](../../api-reference.md#queryrequest-1). The query results will include detailed tracing information to help you identify performance issues.
-> **Note:** Distributed **measure** queries run through the vectorized query path by default, which emits a different span tree (`distributed-…`, `broadcast-agg`, `groupby-agg-map`, `reduce-raw-frames`, …). See [Vectorized Measure Query Tracing](../observability/tracing.md#vectorized-measure-query-tracing) for that span shape plus worked empty-result and slow-query diagnosis walkthroughs. The span names below describe the per-data-node read path — used by stream and trace queries, and by [...]
+> **Note:** Distributed **measure** queries run through the vectorized query path, which emits a different span tree (`distributed-…`, `broadcast-agg`, `groupby-agg-map`, `reduce-raw-frames`, …). See [Vectorized Measure Query Tracing](../observability/tracing.md#vectorized-measure-query-tracing) for that span shape plus worked empty-result and slow-query diagnosis walkthroughs. The span names below describe the per-data-node read path.
There are some important nodes in the trace result:
diff --git a/docs/operation/upgrade.md b/docs/operation/upgrade.md
index dff997afe..d13d837fe 100644
--- a/docs/operation/upgrade.md
+++ b/docs/operation/upgrade.md
@@ -34,9 +34,10 @@ Before upgrading the cluster, you should check CHANGELOG.md](https://github.com/
- After upgrading all nodes, the cluster will be running the new version.
> **Exception — upgrading to 0.11 or later.** The vectorized query paths change the
-> liaison<->data wire format, and a data node must never have a vectorized path enabled
-> that the liaison it answers cannot decode. For 0.11 the order is **reversed**: upgrade
-> "liaison" nodes first, then "data" nodes. See [Vectorized query paths enabled by default](#vectorized-query-paths-enabled-by-default-breaking-for-rolling-upgrades).
+> liaison<->data wire format, and a data node must never emit a frame that the liaison it
+> answers cannot decode. For 0.11 and later the order is **reversed**: upgrade "liaison"
+> nodes first, then "data" nodes. See [Vectorized query paths enabled by default](#vectorized-query-paths-enabled-by-default-breaking-for-rolling-upgrades)
+> for 0.11 and [Row-based query path removed](#row-based-query-path-removed-breaking) for 0.12.
To ensure this strategy works, you should have a minimum of one node for each role of node in the cluster. For example, if you have a 2-node cluster, you should have at least one "liaison" and one "data" node.
@@ -55,6 +56,43 @@ The data ingestion and retrieval will be stopped during the upgrade process. The
If you don't have enough resource to perform a rolling upgrade or you have a large cluster with many nodes, you can use the minimum downtime strategy.
+## Upgrading to 0.12
+
+This section describes breaking changes and important behavioral changes when upgrading to BanyanDB 0.12.0.
+
+### Row-based query path removed (breaking)
+
+The row-based query execution path is gone. Stream, measure and trace queries all execute through the vectorized columnar pipeline, and a distributed data node always answers with a native columnar frame. There is no runtime switch back to row execution or to the protobuf wire format.
+
+`--stream-vectorized-enabled`, `--trace-vectorized-enabled` and `--measure-vectorized-enabled` are still registered so an existing command line keeps parsing, but they are no longer switches:
+
+- `=true`, or the flag omitted, is the only supported setting and changes nothing.
+- `=false` makes the node **exit at startup** with an error naming [apache/skywalking#13998](https://github.com/apache/skywalking/issues/13998), rather than silently discarding an explicit rollback request.
+
+All three flags are removed outright in 0.13.0. Drop them from systemd units, Helm values, container arguments and compose files as part of this upgrade.
+
+**Upgrade "liaison" nodes BEFORE "data" nodes**, as in 0.11. A 0.12 data node emits columnar frames unconditionally, and a liaison on an older binary has no frame decoder. The 0.11 workaround — starting the new data nodes with the vectorized paths disabled and enabling them once every liaison is upgraded — is no longer available, because the paths can no longer be disabled. If you cannot control node ordering, use the [Minimum Downtime Strategy](#minimum-downtime-strategy-to-upgrade-a-cluster).
+
+**Rollback.** Rollback is a binary downgrade, not a flag flip: follow the [Rollback](#rollback) procedure below. The on-disk format is unchanged by this release, so no data migration is involved.
+
+### Stream timestamp-order queries with criteria spanning multiple segments
+
+Stream queries that are ordered by timestamp, carry a criteria filter, and span more than one segment may now return fewer elements than 0.11.x did. The removed row scan resumed segment by segment: after its first `maxElementSize`-sized batch it could keep pulling from later segments, accumulating further matches until the requested limit was filled. The vectorized scan does not see segment boundaries, so it caps at `maxElementSize` and stops there.
+
+The results it returns are correct and correctly ordered — the response is simply under-filled against the limit where the old path would have gone back for more. Index-order (tag-ordered) queries are unaffected: their merge stays uncapped and still fills the limit.
+
+A client that needs the missing elements should re-issue the query over a narrower time range or page forward. This divergence was always documented as one the vectorized path deliberately does not emulate; with the row path gone, the vectorized answer is now the only answer.
+
+### Index-order queries that do not project the ordered tag
+
+An index-order stream query whose ordered tag is not in the client's tag projection now answers over protobuf on a distributed data node instead of over a columnar frame. The scan projects that tag internally to derive its sort key and hides it again before egress, while the frame egress rebuilds its projection from the batch schema and would leak the hidden tag — so this query shape is routed onto the protobuf path. Results are unaffected; the query simply forgoes the raw-wire fast path.
+
+### Stream query shapes with no execution path (breaking)
+
+A stream query the vectorized engine cannot plan now fails with an error. The row path served these shapes without complaint, so the query returned results instead.
+
+One shape reaches this: an `orderBy` naming a tag that no longer resolves against the stream's schema, such as an index rule left behind after its tag was dropped. The error names the reason. Drop the stale index rule, or order by a tag that still exists.
+
## Upgrading to 0.11
This section describes breaking changes and important behavioral changes when upgrading to BanyanDB 0.11.0.
@@ -105,6 +143,8 @@ Standalone deployments are unaffected: the frame is only emitted on a distribute
**Rollback.** Pass `--stream-vectorized-enabled=false` / `--trace-vectorized-enabled=false` / `--measure-vectorized-enabled=false` on the standalone or data-node command line and restart. The row path and the protobuf wire format resume immediately; no data migration is involved, because the flags affect only the query and wire paths, never the on-disk format.
+> This rollback applies to 0.11 only. The row path was removed in 0.12.0 and `=false` is no longer accepted — see [Row-based query path removed](#row-based-query-path-removed-breaking).
+
## Upgrading to 0.10
This section describes breaking changes and important behavioral changes when upgrading to BanyanDB 0.10.0.
diff --git a/docs/soak/g5d-phase-d-summary.md b/docs/soak/g5d-phase-d-summary.md
index b0f68ce04..0b2c93693 100644
--- a/docs/soak/g5d-phase-d-summary.md
+++ b/docs/soak/g5d-phase-d-summary.md
@@ -7,6 +7,13 @@
**Host:** 32-core Linux, 31 GB RAM, no swap; Docker daemon 29.3.1
**Harness:** `scripts/soak-vectorized.sh` + `scripts/soak-monitor.sh` + `cmd/soak-driver`
+> **Historical record — superseded by 0.12.0.** This document describes the vectorized-query
+> rollout gate as it ran while the row-based query path still existed. The
+> `--measure-vectorized-enabled` flag it toggles no longer selects a query engine: the row path
+> was removed in 0.12.0 ([apache/skywalking#13998](https://github.com/apache/skywalking/issues/13998)),
+> the flag is a no-op that fails fast on `=false`, and it is removed entirely in 0.13.0. The
+> procedure is preserved verbatim for provenance — do not run it against a 0.12.0 or later build.
+
## Acceptance criteria
### 1. ≥48 h staging run with `--measure-vectorized-enabled=true` — **PASS**
diff --git a/docs/soak/g5d-runbook.md b/docs/soak/g5d-runbook.md
index fd4995096..85867f69c 100644
--- a/docs/soak/g5d-runbook.md
+++ b/docs/soak/g5d-runbook.md
@@ -3,6 +3,13 @@
This document covers how to run and interpret the G5d soak test for the
vectorized-query path in SkyWalking BanyanDB.
+> **Historical record — superseded by 0.12.0.** This document describes the vectorized-query
+> rollout gate as it ran while the row-based query path still existed. The
+> `--measure-vectorized-enabled` flag it toggles no longer selects a query engine: the row path
+> was removed in 0.12.0 ([apache/skywalking#13998](https://github.com/apache/skywalking/issues/13998)),
+> the flag is a no-op that fails fast on `=false`, and it is removed entirely in 0.13.0. The
+> procedure is preserved verbatim for provenance — do not run it against a 0.12.0 or later build.
+
## Prerequisites
- Docker daemon running and reachable (`docker info` succeeds).
diff --git a/docs/soak/g9f-runbook.md b/docs/soak/g9f-runbook.md
index 3c6135a17..652bb1d96 100644
--- a/docs/soak/g9f-runbook.md
+++ b/docs/soak/g9f-runbook.md
@@ -1,5 +1,12 @@
# G9f Throughout-Vec Measure Query — Operator Runbook
+> **Historical record — superseded by 0.12.0.** This document describes the vectorized-query
+> rollout gate as it ran while the row-based query path still existed. The
+> `--measure-vectorized-enabled` flag it toggles no longer selects a query engine: the row path
+> was removed in 0.12.0 ([apache/skywalking#13998](https://github.com/apache/skywalking/issues/13998)),
+> the flag is a no-op that fails fast on `=false`, and it is removed entirely in 0.13.0. The
+> procedure is preserved verbatim for provenance — do not run it against a 0.12.0 or later build.
+
## What G9f does
G9f makes vec the sole production path for distributed measure queries. A
diff --git a/docs/soak/trace-soak-runbook.md b/docs/soak/trace-soak-runbook.md
index 549465249..bc93491e1 100644
--- a/docs/soak/trace-soak-runbook.md
+++ b/docs/soak/trace-soak-runbook.md
@@ -118,11 +118,10 @@ SOAK_ENGINE=trace \
./scripts/soak-vectorized.sh
```
-### Phase 0 — vec-off, capture golden baseline
+### Phase 0 — capture golden baseline
-1. `BANYANDB_VEC_ENABLED=false docker compose up -d banyandb` — standalone
- BanyanDB with `--trace-vectorized-enabled=false`, resource-limited (2 GB /
- 2 CPU).
+1. `docker compose up -d banyandb` — standalone BanyanDB, resource-limited
+ (2 GB / 2 CPU).
2. Health-check `http://localhost:17913/api/healthz` (120-attempt cap, 5 s
intervals).
3. `docker compose run --rm soak-driver seed-fixture --engine trace
@@ -142,17 +141,14 @@ SOAK_ENGINE=trace \
if any catalog query returns zero traces** (engine-agnostic guard).
6. Stop BanyanDB → `cp -a data/. data-snapshot/` → `docker compose down -v`.
-### Phase 1 — vec-on, soak
+### Phase 1 — soak
-1. Restore snapshot → `BANYANDB_VEC_ENABLED=true docker compose up -d banyandb`.
-2. Health-check; then **vec-flag-honored gate**: grep startup logs for
- `trace-vectorized-enabled=true`. This grep is load-bearing because the
- parity-clean S=5 fixture produces identical vec-on/vec-off output by design,
- so parity alone cannot distinguish "vec engaged" from "vec silently disabled".
-3. Initial pprof-grab (heap + goroutine) → `pprof-start/`.
-4. Background loops (fail-tolerant — WARN on a bad tick, never abort):
+1. Restore snapshot → `docker compose up -d banyandb`; health-check.
+2. Initial pprof-grab (heap + goroutine) → `pprof-start/`.
+3. Background loops (fail-tolerant — WARN on a bad tick, never abort):
- **parity** (every `PARITY_INTERVAL_MIN`): `replay-and-diff --engine trace`
- re-runs the catalog vs vec-on live data; writes `diff-<ts>.json`.
+ re-runs the catalog against the live post-restore data; writes
+ `diff-<ts>.json`.
- **pprof** (every `PPROF_INTERVAL_MIN`): heap + goroutine snapshots.
- **write-load** (continuous, rate-capped at `SOAK_WRITE_RPS` spans/s):
deterministic writes into `bench-trace-load` using timestamps offset 365
@@ -162,7 +158,7 @@ SOAK_ENGINE=trace \
- **log capture** (rotation-bounded `--since 60s` snapshots → `banyand.log`)
and memory-alert grep
(`budget|MemoryTracker|panic|vectorized` → `memory-alerts.log`).
-5. Wait `SOAK_HOURS` → kill loops → final pprof-grab (`pprof-end/`) → final
+4. Wait `SOAK_HOURS` → kill loops → final pprof-grab (`pprof-end/`) → final
`replay-and-diff` (sets `FINAL_PASS`) → write `summary.json`.
### Tapered monitor (recommended for unattended runs)
@@ -234,7 +230,7 @@ test/integration/distributed/querybench/run-docker.sh
**Phase 1 — parity fixture (200 iterations, liveness + leak gate)**
-- Boots in-process cluster vec-on; seeds 1000-trace × 20-span uniform fixture
+- Boots an in-process cluster; seeds 1000-trace × 20-span uniform fixture
(budget 256 MiB, never truncates → parity clean).
- Warmup: 3 query iterations to stabilize JIT paths and GC.
- Snapshots post-warmup `vtrace.QueryCount()` + `runtime.ReadMemStats.HeapInuse`.
@@ -272,7 +268,6 @@ test/integration/distributed/querybench/run-docker.sh
| **No crash/restart** | BanyanDB container uptime continuous; `banyand.log` not stale (`soak-monitor.sh`) |
| **No memory-alert lines** | `memory-alerts.log` has 0 lines matching `budget\|MemoryTracker\|panic\|vectorized` |
| **Write-load alive** | `write_load_spans > 0` in `summary.json` |
-| **Vec flag honored** | Startup log grep confirms `--trace-vectorized-enabled=true` was honored |
### Advisory signals (human review only, not gates)
@@ -320,7 +315,7 @@ All artifacts land under `dist/soak/<YYYYMMDDTHHMMSS>/`:
dist/soak/<ts>/
run.log # tee'd full orchestrator log
baseline.json # array of traceBaselineRecord (proto-JSON Traces)
- data-snapshot/ # raw BanyanDB /data dir from Phase 0 (vec-off)
+ data-snapshot/ # raw BanyanDB /data dir from Phase 0
pprof-start/
heap-<unix>.pb.gz # gzip pprof heap
goroutine-<unix>.txt # goroutine dump (debug=1 text format)
@@ -345,7 +340,6 @@ dist/soak/<ts>/
"soak_hours": 48,
"t1_ms": 1704067200000,
"final_parity_pass": true,
- "vec_flag_honored": true,
"goroutine_count_start": 120,
"goroutine_count_end": 122,
"memory_alert_lines": 0,
@@ -389,8 +383,8 @@ dist/soak/<ts>/
### Quick all-green check
```bash
-# Instrument 1: final parity + vec flag + write-load + alerts
-jq '{final_parity_pass, vec_flag_honored, write_load_alive, memory_alert_lines,
+# Instrument 1: final parity + write-load + alerts
+jq '{final_parity_pass, write_load_alive, memory_alert_lines,
goroutine_count_start, goroutine_count_end}' dist/soak/<ts>/summary.json
# Instrument 1: all intermediate diffs
@@ -430,22 +424,27 @@ jq '.divergences' dist/soak/<ts>/diff-<ts>.json
```
**Parity-FAIL disambiguation**: a divergence after Phase 1 could be caused by
-the vec reader or by a merge/data artifact on the Phase-1 data. To distinguish:
+the reader or by a merge/data artifact accumulated on the Phase-1 data. There is
+no second query engine to cross-check against — the row path was removed in
+0.12.0 — so the control is the Phase-0 snapshot, which the soak has not written
+to. Restore it into a scratch data dir and replay the same catalog:
```bash
-# Re-run the failing catalog query with vec-off on Phase-1 data:
-BANYANDB_VEC_ENABLED=false \
+# Replay the failing catalog query against the untouched Phase-0 snapshot:
+rm -rf /tmp/soak-replay && cp -a dist/soak/<ts>/data-snapshot /tmp/soak-replay
+SOAK_DATA_DIR=/tmp/soak-replay \
docker compose -f test/soak/docker-compose.soak.yaml up -d banyandb
docker compose -f test/soak/docker-compose.soak.yaml run --rm soak-driver \
replay-and-diff --engine trace \
--addr banyandb:17912 \
--catalog /catalog/trace.json \
--baseline /artifacts/baseline.json \
- --report /artifacts/diff-vec-off-debug.json
+ --report /artifacts/diff-snapshot-debug.json
```
-If `diff-vec-off-debug.json` also diverges, the cause is in the
-data/merge path, not the vec reader. If it passes, the vec reader is at fault.
+If `diff-snapshot-debug.json` passes, the baseline data still reads correctly and
+the divergence came from what Phase 1 did to the data — merge, compaction or
+expiry. If it also diverges, the reader is at fault.
### Memory-alert lines
@@ -466,10 +465,9 @@ bug against the vectorized trace pipeline with the diff attached.
**Symptom**: `liveness_pass: false` — `query_count_delta < 200`.
-**Meaning**: the vec query path was not reached on every iteration. This can
-happen if the `--trace-vectorized-enabled` flag was not honored in-process, or
-if a dispatch branch fell through to the row path. Examine the test's `t.Log`
-output for iteration-level error messages.
+**Meaning**: the query path was not reached on every iteration — iterations are
+erroring out before they query. Examine the test's `t.Log` output for
+iteration-level error messages.
### Instrument 2 heap-leak gate FAIL
@@ -510,20 +508,14 @@ A stall means no flush/merge/compaction stress during Phase 1.
DQB_SOAK=1 DQB_ENGINE=trace ./test/integration/distributed/querybench/run-docker.sh --cpus 4 --memory 8g
```
Both should complete without errors, `summary.json` should have
- `final_parity_pass: true` and `vec_flag_honored: true`;
+ `final_parity_pass: true`;
`soak-trace-vec-result.json` should have all pass fields `true`.
2. **Parity has teeth** (negative control): inject one extra span into the
Phase-1 fixture and re-run `replay-and-diff` — `diff-final.json` must show
`"pass": false`.
-3. **Vec-engaged confirmation** (separate run): run the soak with a deliberately
- divergent fixture (e.g. extra span on the vec-on side only) and confirm
- `diff-final.json` FAILs. The parity-clean S=5 fixture cannot distinguish
- vec-engaged from vec-silently-disabled by result content alone; this
- divergent-fixture run corroborates the startup-log grep gate.
-
-4. **Measure regression**: `SOAK_ENGINE=measure SMOKE=1 ./scripts/soak-vectorized.sh`
+3. **Measure regression**: `SOAK_ENGINE=measure SMOKE=1 ./scripts/soak-vectorized.sh`
must still pass unchanged (measure path is byte-for-byte unmodified).
## Named v1 Gap — Production Distributed-Topology Soak
@@ -544,15 +536,11 @@ covering real network fan-out, on-wire frames, and multi-node container RSS.
The in-process wire/merge path is already covered for correctness; what remains
is the production transport layer.
-## Rollback
-
-To re-run Phase 1 with vec disabled (reproducing baseline behavior):
+## Teardown
-```bash
-BANYANDB_VEC_ENABLED=false \
-SOAK_DATA_DIR=./test/soak/data \
-docker compose -f test/soak/docker-compose.soak.yaml up -d banyandb
-```
+The row-based query path was removed in 0.12.0
+([apache/skywalking#13998](https://github.com/apache/skywalking/issues/13998)),
+so there is no engine to roll back to; a run is torn down, not reverted.
Tear down with:
diff --git a/pkg/panicdiag/lintrawgo/baseline.txt b/pkg/panicdiag/lintrawgo/baseline.txt
index e330da371..2d51dd8e4 100644
--- a/pkg/panicdiag/lintrawgo/baseline.txt
+++ b/pkg/panicdiag/lintrawgo/baseline.txt
@@ -119,7 +119,6 @@ banyand/queue/sub/group_lifecycle_cold_tier_test.go:289
banyand/queue/test/chunked_sync_common.go:102
banyand/stream/block_scanner_test.go:138
banyand/stream/part.go:296
-banyand/stream/query_by_idx.go:139
banyand/stream/tstable_test.go:103
banyand/stream/tstable_test.go:245
banyand/trace/file_part_sync_test.go:383
diff --git a/pkg/query/executor/interface.go b/pkg/query/executor/interface.go
index 12d789509..d210bdd96 100644
--- a/pkg/query/executor/interface.go
+++ b/pkg/query/executor/interface.go
@@ -38,7 +38,6 @@ import (
// surfaces the engine-side --stream-vectorized-* configuration so the vec
// dispatch can decide whether the flag is on.
type StreamExecutionContext interface {
- Query(ctx context.Context, opts model.StreamQueryOptions) (model.StreamQueryResult, error)
QueryVectorized(ctx context.Context, opts model.StreamQueryOptions) (StreamVecScanSource, error)
VectorizedConfig() vstream.VectorizedConfig
}
@@ -56,17 +55,23 @@ type StreamVecScanSource interface {
Release()
}
+// StreamCloser releases the resources a stream plan node holds. It is separate
+// from StreamExecutable so a plan tree can be released without asserting the row
+// Execute capability.
+type StreamCloser interface {
+ Close()
+}
+
// StreamExecutable allows querying in the stream schema.
type StreamExecutable interface {
Execute(context.Context) ([]*streamv1.Element, error)
- Close()
+ StreamCloser
}
-// StreamVecExecutable is the optional capability a stream plan node exposes when
-// it can be executed through the native columnar (vectorized) path. The data-node
-// processor type-asserts the analyzed plan to this interface; a plan shape that
-// cannot be vectorized (multi-group merge, skipping filter, tag filter, etc.)
-// simply does not implement it, so the assertion fails and the row path runs.
+// StreamVecExecutable is the capability a stream plan node exposes when it can be
+// executed through the native columnar (vectorized) path. The data-node processor
+// type-asserts the analyzed plan to this interface; since the row path was removed
+// in 0.12.0 a failed assertion is a hard error, not a fallback.
//
// ExecuteVectorized returns the fully merged/deduped/limited columnar batches
// (the M4 pipeline already applied) plus the batch schema, so both the
@@ -77,6 +82,11 @@ type StreamVecExecutable interface {
// ProjectionTags returns the projected tag families/names in projection order,
// so the egress builds the same tag families/tags the row path would.
ProjectionTags() []model.TagProjection
+ // HidesOrderTag reports whether the scan requests an ordered tag that is absent
+ // from ProjectionTags(). The columnar frame egress rebuilds the projection from
+ // the batch schema, so such a scan must take the proto egress or the hidden tag
+ // leaks into the result.
+ HidesOrderTag() bool
}
// MeasureExecutionContext allows retrieving data through the measure module.
diff --git a/pkg/query/logical/measure/measure_analyzer.go b/pkg/query/logical/measure/measure_analyzer.go
deleted file mode 100644
index 8a81122be..000000000
--- a/pkg/query/logical/measure/measure_analyzer.go
+++ /dev/null
@@ -1,246 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package measure
-
-import (
- "fmt"
- "math"
- "time"
-
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- "github.com/apache/skywalking-banyandb/pkg/query/executor"
- "github.com/apache/skywalking-banyandb/pkg/query/logical"
-)
-
-const defaultLimit uint32 = 100
-
-// BuildSchema returns Schema loaded from the metadata repository.
-//
-// Deprecated: row-path measure plan; see .omc/g8-plan.md. The vec
-// measure analyzer (pkg/query/vectorized/measure/plan.Analyze) consumes
-// *databasev1.Measure directly via measure.BuildBatchSchema and does
-// not produce a logical.Schema.
-func BuildSchema(md *databasev1.Measure, indexRules []*databasev1.IndexRule) (logical.Schema, error) {
- md.GetEntity()
-
- ms := &schema{
- common: &logical.CommonSchema{
- IndexRules: indexRules,
- TagSpecMap: make(map[string]*logical.TagSpec),
- EntityList: md.GetEntity().GetTagNames(),
- },
- measure: md,
- fieldMap: make(map[string]*logical.FieldSpec),
- }
-
- ms.common.RegisterTagFamilies(md.GetTagFamilies())
-
- for fieldIdx, spec := range md.GetFields() {
- ms.registerField(fieldIdx, spec)
- }
-
- return ms, nil
-}
-
-// Analyze converts logical expressions to executable operation tree represented by Plan.
-//
-// Deprecated: This is the row-based measure query analyzer. The vec measure
-// subsystem (.omc/g8-plan.md) will replace this with a separate vec analyzer
-// that produces a vec plan tree (no leaf substitution into the row plan).
-// Do not extend with new features; the dispatch from `banyand/query/processor.go`
-// will route to the vec analyzer when `VectorizedConfig.Enabled` is true once
-// G8 lands.
-func Analyze(criteria *measurev1.QueryRequest, metadata []*commonv1.Metadata, ss []logical.Schema,
- ecc []executor.MeasureExecutionContext, emitPartial bool,
-) (logical.Plan, error) {
- if len(metadata) != len(ss) {
- return nil, fmt.Errorf("number of schemas %d not equal to metadata count %d", len(ss), len(metadata))
- }
- groupByEntity := false
- var groupByTags [][]*logical.Tag
- if criteria.GetGroupBy() != nil {
- groupByProjectionTags := criteria.GetGroupBy().GetTagProjection()
- groupByTags = make([][]*logical.Tag, len(groupByProjectionTags.GetTagFamilies()))
- tags := make([]string, 0)
- for i, tagFamily := range groupByProjectionTags.GetTagFamilies() {
- groupByTags[i] = logical.NewTags(tagFamily.GetName(), tagFamily.GetTags()...)
- tags = append(tags, tagFamily.GetTags()...)
- }
- if logical.StringSlicesEqual(ss[0].EntityList(), tags) {
- groupByEntity = true
- }
- }
- var plan logical.UnresolvedPlan
- var s logical.Schema
- tagProjection := logical.ToTags(criteria.GetTagProjection())
- if len(metadata) == 1 {
- plan = parseFields(criteria, metadata[0], ecc[0], groupByEntity, tagProjection)
- s = ss[0]
- } else {
- var err error
- if s, err = mergeSchema(ss); err != nil {
- return nil, err
- }
- plan = &unresolvedMerger{
- criteria: criteria,
- metadata: metadata,
- ecc: ecc,
- tagProjection: tagProjection,
- groupByEntity: groupByEntity,
- }
- }
-
- ms := s.(*schema)
- if len(tagProjection) > 0 {
- if err := ms.common.ValidateProjectionTags(tagProjection...); err != nil {
- return nil, err
- }
- }
- fieldProjection := criteria.GetFieldProjection().GetNames()
- if len(fieldProjection) > 0 {
- if err := ms.ValidateProjectionFields(fieldProjection...); err != nil {
- return nil, err
- }
- }
-
- // parse limit and offset
- limitParameter := criteria.GetLimit()
- if limitParameter == 0 {
- limitParameter = defaultLimit
- }
- pushedLimit := int(limitParameter + criteria.GetOffset())
-
- if criteria.GetGroupBy() != nil {
- plan = newUnresolvedGroupBy(plan, groupByTags, groupByEntity)
- pushedLimit = math.MaxInt
- }
-
- if criteria.GetAgg() != nil {
- plan = newUnresolvedAggregation(plan,
- logical.NewField(criteria.GetAgg().GetFieldName()),
- criteria.GetAgg().GetFunction(),
- criteria.GetGroupBy() != nil,
- emitPartial,
- false,
- )
- pushedLimit = math.MaxInt
- }
-
- if criteria.GetTop() != nil {
- plan = top(plan, criteria.GetTop())
- }
-
- plan = limit(plan, criteria.GetOffset(), limitParameter)
- p, err := plan.Analyze(s)
- if err != nil {
- return nil, err
- }
- rules := []logical.OptimizeRule{
- logical.NewPushDownOrder(criteria.OrderBy),
- logical.NewPushDownMaxSize(pushedLimit),
- }
- if err := logical.ApplyRules(p, rules...); err != nil {
- return nil, err
- }
- return p, nil
-}
-
-// DistributedAnalyze converts logical expressions to executable operation tree represented by Plan.
-//
-// Deprecated: Row-based distributed-query analyzer; see Analyze for the
-// replacement plan. The vec measure subsystem (G8) will provide a
-// vec-distributed variant once standalone-vec parity is stable.
-func DistributedAnalyze(criteria *measurev1.QueryRequest, ss []logical.Schema, broadcastTimeout time.Duration) (logical.Plan, error) {
- var groupByTags [][]*logical.Tag
- if criteria.GetGroupBy() != nil {
- groupByProjectionTags := criteria.GetGroupBy().GetTagProjection()
- groupByTags = make([][]*logical.Tag, len(groupByProjectionTags.GetTagFamilies()))
- for i, tagFamily := range groupByProjectionTags.GetTagFamilies() {
- groupByTags[i] = logical.NewTags(tagFamily.GetName(), tagFamily.GetTags()...)
- }
- }
-
- pushDownAgg := criteria.GetAgg() != nil
- plan := newUnresolvedDistributed(criteria, pushDownAgg, broadcastTimeout)
-
- // parse limit and offset
- limitParameter := criteria.GetLimit()
- if limitParameter == 0 {
- limitParameter = defaultLimit
- }
- pushedLimit := int(limitParameter + criteria.GetOffset())
-
- if criteria.GetGroupBy() != nil {
- plan = newUnresolvedGroupBy(plan, groupByTags, false)
- pushedLimit = math.MaxInt
- }
-
- if criteria.GetAgg() != nil {
- plan = newUnresolvedAggregation(plan,
- logical.NewField(criteria.GetAgg().GetFieldName()),
- criteria.GetAgg().GetFunction(),
- criteria.GetGroupBy() != nil,
- false, // emitPartial: liaison does not emit partial
- pushDownAgg, // reduceMode: reduce partials from data nodes when push-down is active
- )
- pushedLimit = math.MaxInt
- }
-
- if criteria.GetTop() != nil {
- plan = top(plan, criteria.GetTop())
- }
-
- plan = limit(plan, criteria.GetOffset(), limitParameter)
-
- var s logical.Schema
- var err error
- if len(ss) == 1 {
- s = ss[0]
- } else {
- if s, err = mergeSchema(ss); err != nil {
- return nil, err
- }
- }
-
- p, err := plan.Analyze(s)
- if err != nil {
- return nil, err
- }
- rules := []logical.OptimizeRule{
- logical.NewPushDownOrder(criteria.OrderBy),
- logical.NewPushDownMaxSize(pushedLimit),
- }
- if err := logical.ApplyRules(p, rules...); err != nil {
- return nil, err
- }
- return p, nil
-}
-
-func parseFields(criteria *measurev1.QueryRequest, metadata *commonv1.Metadata, ec executor.MeasureExecutionContext,
- groupByEntity bool, tagProjection [][]*logical.Tag,
-) logical.UnresolvedPlan {
- projFields := make([]*logical.Field, len(criteria.GetFieldProjection().GetNames()))
- for i, fieldNameProj := range criteria.GetFieldProjection().GetNames() {
- projFields[i] = logical.NewField(fieldNameProj)
- }
- timeRange := criteria.GetTimeRange()
- return indexScan(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
- tagProjection, projFields, groupByEntity, criteria.GetCriteria(), ec)
-}
diff --git a/pkg/query/logical/measure/measure_plan.go b/pkg/query/logical/measure/measure_plan.go
deleted file mode 100644
index 39fd2b134..000000000
--- a/pkg/query/logical/measure/measure_plan.go
+++ /dev/null
@@ -1,119 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package measure
-
-import (
- "context"
- "fmt"
-
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- "github.com/apache/skywalking-banyandb/pkg/query/executor"
- "github.com/apache/skywalking-banyandb/pkg/query/logical"
-)
-
-var (
- _ logical.Plan = (*limitPlan)(nil)
- _ logical.UnresolvedPlan = (*limitPlan)(nil)
-)
-
-// Deprecated: row-path measure plan; see .omc/g8-plan.md. The vec
-// subsystem replaces row-path limit/offset paging via plan.Limit at
-// pkg/query/vectorized/measure/plan/limit.go.
-type limitPlan struct {
- *logical.Parent
- offset uint32
- limit uint32
-}
-
-func (l *limitPlan) Execute(ec context.Context) (executor.MIterator, error) {
- dps, err := l.Parent.Input.(executor.MeasureExecutable).Execute(ec)
- if err != nil {
- return nil, err
- }
... 8066 lines suppressed ...