You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/09/29 13:19:29 UTC
[skywalking-banyandb] branch main updated: Refactor liaison and
other relevant components (#53)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 9c3fc43 Refactor liaison and other relevant components (#53)
9c3fc43 is described below
commit 9c3fc4341d36f9eec7626dd59b0d4413ca268cd5
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Sep 29 21:19:23 2021 +0800
Refactor liaison and other relevant components (#53)
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
api/common/id.go | 1 -
api/common/metadata.go | 52 ---
api/data/stream.go | 22 +-
api/data/trace.go | 64 ---
api/event/discovery.go | 25 +-
api/proto/banyandb/database/v2/event.pb.go | 205 ++++++---
api/proto/banyandb/database/v2/event.proto | 12 +-
api/proto/banyandb/model/v2/common.pb.go | 110 ++++-
api/proto/banyandb/model/v2/common.proto | 4 +
api/proto/banyandb/stream/v2/write.pb.go | 133 +++---
api/proto/banyandb/stream/v2/write.proto | 11 +-
api/schema/index.go | 36 --
api/schema/series.go | 30 --
banyand/index/index.go | 289 ------------
banyand/index/index_test.go | 181 --------
banyand/index/search.go | 356 ---------------
banyand/index/search_test.go | 255 -----------
banyand/index/tsdb/field_map.go | 64 ---
banyand/index/tsdb/mem.go | 91 ----
banyand/index/tsdb/mem_test.go | 261 -----------
banyand/index/tsdb/term_map.go | 143 ------
banyand/index/tsdb/tsdb.go | 65 ---
banyand/internal/cmd/standalone.go | 3 +-
banyand/liaison/grpc/discovery.go | 126 ++++++
banyand/liaison/grpc/server.go | 139 ++++++
banyand/liaison/grpc/stream.go | 84 ++++
banyand/liaison/grpc/stream_test.go | 259 +++++++++++
banyand/liaison/grpc/trace.go | 347 --------------
banyand/liaison/grpc/trace_test.go | 299 ------------
banyand/query/v1/processor.go | 102 -----
banyand/query/v1/processor_test.go | 504 ---------------------
banyand/query/v1/query.go | 45 --
banyand/query/v2/processor.go | 2 +-
banyand/query/v2/processor_test.go | 12 +-
banyand/series/schema/schema.go | 44 --
banyand/series/schema/sw/index_rule.bin | 16 -
banyand/series/schema/sw/index_rule.textproto | 89 ----
banyand/series/schema/sw/index_rule_binding.bin | 5 -
.../series/schema/sw/index_rule_binding.textproto | 31 --
banyand/series/schema/sw/sw.go | 126 ------
banyand/series/schema/sw/trace_series.bin | 23 -
banyand/series/schema/sw/trace_series.textproto | 89 ----
banyand/series/series.go | 90 ----
banyand/series/trace/common_test.go | 258 -----------
banyand/series/trace/query.go | 279 ------------
banyand/series/trace/query_test.go | 321 -------------
banyand/series/trace/schema.go | 102 -----
banyand/series/trace/schema_test.go | 108 -----
banyand/series/trace/service.go | 186 --------
banyand/series/trace/trace.go | 377 ---------------
banyand/series/trace/write.go | 157 -------
banyand/series/trace/write_test.go | 193 --------
banyand/storage/block.go | 113 -----
banyand/storage/database.go | 321 -------------
banyand/storage/database_test.go | 277 -----------
banyand/storage/storage.go | 110 -----
banyand/stream/service.go | 78 +++-
banyand/stream/stream.go | 18 +-
banyand/stream/stream_query.go | 2 +-
banyand/stream/stream_query_test.go | 11 +-
banyand/stream/stream_write.go | 80 +---
banyand/stream/stream_write_test.go | 52 +--
banyand/tsdb/seriesdb.go | 35 +-
banyand/tsdb/seriesdb_test.go | 12 +-
banyand/tsdb/tsdb_test.go | 7 +-
go.mod | 3 -
go.sum | 3 -
pkg/index/inverted/inverted_test.go | 13 +-
pkg/index/lsm/lsm_test.go | 7 +-
pkg/partition/entity.go | 77 ++++
pkg/pb/v2/database.go | 166 -------
pkg/pb/v2/fields.go | 151 ------
pkg/pb/v2/write.go | 169 ++++---
pkg/posting/posting.go | 75 ---
pkg/posting/roaring/roaring.go | 214 ---------
pkg/query/v1/executor/interface.go | 34 --
pkg/query/v1/logical/analyzer.go | 184 --------
pkg/query/v1/logical/analyzer_test.go | 251 ----------
pkg/query/v1/logical/common_test.go | 242 ----------
pkg/query/v1/logical/expr.go | 193 --------
pkg/query/v1/logical/expr_literal.go | 160 -------
pkg/query/v1/logical/interface.go | 68 ---
pkg/query/v1/logical/plan.go | 162 -------
pkg/query/v1/logical/plan_execution_test.go | 242 ----------
pkg/query/v1/logical/plan_indexscan.go | 238 ----------
pkg/query/v1/logical/plan_orderby.go | 160 -------
pkg/query/v1/logical/plan_tablescan.go | 143 ------
pkg/query/v1/logical/plan_traceid.go | 132 ------
pkg/query/v1/logical/schema.go | 151 ------
pkg/query/v2/logical/common_test.go | 9 +-
pkg/query/v2/logical/plan_execution_test.go | 16 +-
pkg/{query/v1/logical/format.go => run/test.go} | 60 ++-
banyand/series/idgen.go => pkg/test/retry.go | 43 +-
pkg/test/space.go | 4 +-
94 files changed, 1327 insertions(+), 9985 deletions(-)
diff --git a/api/common/id.go b/api/common/id.go
index 5442bcc..9964786 100644
--- a/api/common/id.go
+++ b/api/common/id.go
@@ -19,7 +19,6 @@ package common
import "github.com/apache/skywalking-banyandb/pkg/convert"
-type ChunkID uint64
type SeriesID uint64
type ShardID uint32
type ItemID uint64
diff --git a/api/common/metadata.go b/api/common/metadata.go
deleted file mode 100644
index 8f55c1a..0000000
--- a/api/common/metadata.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package common
-
-import (
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
-)
-
-var MetadataKindVersion = KindVersion{Version: "commonv1", Kind: "metadata"}
-
-type Metadata struct {
- KindVersion
- Spec *commonv1.Metadata
-}
-
-func (md Metadata) Equal(other Metadata) bool {
- return md.KindVersion.Kind == other.KindVersion.Kind && md.KindVersion.Version == other.KindVersion.Version &&
- md.Spec.Group == other.Spec.Group &&
- md.Spec.Name == other.Spec.Name
-}
-
-func NewMetadata(spec *commonv1.Metadata) *Metadata {
- return &Metadata{
- KindVersion: MetadataKindVersion,
- Spec: spec,
- }
-}
-
-func NewMetadataByNameAndGroup(name, group string) *Metadata {
- return &Metadata{
- KindVersion: MetadataKindVersion,
- Spec: &commonv1.Metadata{
- Name: name,
- Group: group,
- },
- }
-}
diff --git a/api/data/stream.go b/api/data/stream.go
index 65fc2ef..1bc4a77 100644
--- a/api/data/stream.go
+++ b/api/data/stream.go
@@ -19,26 +19,18 @@ package data
import (
"github.com/apache/skywalking-banyandb/api/common"
- streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
"github.com/apache/skywalking-banyandb/pkg/bus"
)
-var StreamKindVersion = common.KindVersion{Version: "v2", Kind: "data-stream"}
-
-var StreamWriteEventKindVersion = common.KindVersion{
- Version: "v2",
+var StreamWriteKindVersion = common.KindVersion{
+ Version: "v1",
Kind: "stream-write",
}
-var TopicStreamWriteEvent = bus.UniTopic(StreamWriteEventKindVersion.String())
-
-type Stream struct {
- common.KindVersion
- Entities []Entity
-}
+var TopicStreamWrite = bus.UniTopic(StreamWriteKindVersion.String())
-type StreamWriteData struct {
- ShardID uint
- SeriesID uint64
- WriteRequest *streamv2.WriteRequest
+var StreamQueryKindVersion = common.KindVersion{
+ Version: "v1",
+ Kind: "stream-query",
}
+var TopicStreamQuery = bus.BiTopic(StreamQueryKindVersion.String())
diff --git a/api/data/trace.go b/api/data/trace.go
deleted file mode 100644
index 19ac991..0000000
--- a/api/data/trace.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package data
-
-import (
- "github.com/apache/skywalking-banyandb/api/common"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/pkg/bus"
-)
-
-var TraceKindVersion = common.KindVersion{Version: "v1", Kind: "data-trace"}
-
-var WriteEventKindVersion = common.KindVersion{
- Version: "v1",
- Kind: "trace-write",
-}
-var TopicWriteEvent = bus.UniTopic(WriteEventKindVersion.String())
-
-var QueryEventKindVersion = common.KindVersion{
- Version: "v1",
- Kind: "trace-query",
-}
-var TopicQueryEvent = bus.BiTopic(QueryEventKindVersion.String())
-
-type Trace struct {
- common.KindVersion
- Entities []Entity
-}
-
-type Entity struct {
- *tracev1.Entity
-}
-
-type EntityValue struct {
- *tracev1.EntityValue
-}
-type TraceWriteDate struct {
- ShardID uint
- SeriesID uint64
- WriteRequest *tracev1.WriteRequest
-}
-type Write struct {
- common.KindVersion
- Payload *TraceWriteDate
-}
-
-func NewTrace() *Trace {
- return &Trace{KindVersion: TraceKindVersion}
-}
diff --git a/api/event/discovery.go b/api/event/discovery.go
index db9695e..b419442 100644
--- a/api/event/discovery.go
+++ b/api/event/discovery.go
@@ -19,7 +19,6 @@ package event
import (
"github.com/apache/skywalking-banyandb/api/common"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/bus"
)
@@ -30,27 +29,9 @@ var (
}
TopicShardEvent = bus.UniTopic(ShardEventKindVersion.String())
- SeriesEventKindVersion = common.KindVersion{
+ EntityEventKindVersion = common.KindVersion{
Version: "v1",
- Kind: "event-series",
+ Kind: "event-entity",
}
- TopicSeriesEvent = bus.UniTopic(SeriesEventKindVersion.String())
-
- IndexRuleKindVersion = common.KindVersion{Version: "v1", Kind: "index-rule"}
- TopicIndexRule = bus.UniTopic(IndexRuleKindVersion.String())
+ TopicEntityEvent = bus.UniTopic(EntityEventKindVersion.String())
)
-
-type Shard struct {
- common.KindVersion
- Payload v1.ShardEvent
-}
-
-type Series struct {
- common.KindVersion
- Payload v1.SeriesEvent
-}
-
-type IndexRule struct {
- common.KindVersion
- Payload *v1.IndexRuleEvent
-}
diff --git a/api/proto/banyandb/database/v2/event.pb.go b/api/proto/banyandb/database/v2/event.pb.go
index 3c88a6f..f9bb88c 100644
--- a/api/proto/banyandb/database/v2/event.pb.go
+++ b/api/proto/banyandb/database/v2/event.pb.go
@@ -153,19 +153,19 @@ func (x *ShardEvent) GetTime() *timestamppb.Timestamp {
return nil
}
-type SeriesEvent struct {
+type EntityEvent struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Series *v2.Metadata `protobuf:"bytes,1,opt,name=series,proto3" json:"series,omitempty"`
- TagsNamesCompositeSeriesId []string `protobuf:"bytes,2,rep,name=tags_names_composite_series_id,json=tagsNamesCompositeSeriesId,proto3" json:"tags_names_composite_series_id,omitempty"`
- Action Action `protobuf:"varint,3,opt,name=action,proto3,enum=banyandb.database.v2.Action" json:"action,omitempty"`
- Time *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=time,proto3" json:"time,omitempty"`
+ Subject *v2.Metadata `protobuf:"bytes,1,opt,name=subject,proto3" json:"subject,omitempty"`
+ EntityLocator []*EntityEvent_TagLocator `protobuf:"bytes,2,rep,name=entity_locator,json=entityLocator,proto3" json:"entity_locator,omitempty"`
+ Action Action `protobuf:"varint,3,opt,name=action,proto3,enum=banyandb.database.v2.Action" json:"action,omitempty"`
+ Time *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=time,proto3" json:"time,omitempty"`
}
-func (x *SeriesEvent) Reset() {
- *x = SeriesEvent{}
+func (x *EntityEvent) Reset() {
+ *x = EntityEvent{}
if protoimpl.UnsafeEnabled {
mi := &file_banyandb_database_v2_event_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -173,13 +173,13 @@ func (x *SeriesEvent) Reset() {
}
}
-func (x *SeriesEvent) String() string {
+func (x *EntityEvent) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*SeriesEvent) ProtoMessage() {}
+func (*EntityEvent) ProtoMessage() {}
-func (x *SeriesEvent) ProtoReflect() protoreflect.Message {
+func (x *EntityEvent) ProtoReflect() protoreflect.Message {
mi := &file_banyandb_database_v2_event_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -191,39 +191,94 @@ func (x *SeriesEvent) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use SeriesEvent.ProtoReflect.Descriptor instead.
-func (*SeriesEvent) Descriptor() ([]byte, []int) {
+// Deprecated: Use EntityEvent.ProtoReflect.Descriptor instead.
+func (*EntityEvent) Descriptor() ([]byte, []int) {
return file_banyandb_database_v2_event_proto_rawDescGZIP(), []int{1}
}
-func (x *SeriesEvent) GetSeries() *v2.Metadata {
+func (x *EntityEvent) GetSubject() *v2.Metadata {
if x != nil {
- return x.Series
+ return x.Subject
}
return nil
}
-func (x *SeriesEvent) GetTagsNamesCompositeSeriesId() []string {
+func (x *EntityEvent) GetEntityLocator() []*EntityEvent_TagLocator {
if x != nil {
- return x.TagsNamesCompositeSeriesId
+ return x.EntityLocator
}
return nil
}
-func (x *SeriesEvent) GetAction() Action {
+func (x *EntityEvent) GetAction() Action {
if x != nil {
return x.Action
}
return Action_ACTION_UNSPECIFIED
}
-func (x *SeriesEvent) GetTime() *timestamppb.Timestamp {
+func (x *EntityEvent) GetTime() *timestamppb.Timestamp {
if x != nil {
return x.Time
}
return nil
}
+type EntityEvent_TagLocator struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ FamilyOffset uint32 `protobuf:"varint,1,opt,name=family_offset,json=familyOffset,proto3" json:"family_offset,omitempty"`
+ TagOffset uint32 `protobuf:"varint,2,opt,name=tag_offset,json=tagOffset,proto3" json:"tag_offset,omitempty"`
+}
+
+func (x *EntityEvent_TagLocator) Reset() {
+ *x = EntityEvent_TagLocator{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_banyandb_database_v2_event_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *EntityEvent_TagLocator) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EntityEvent_TagLocator) ProtoMessage() {}
+
+func (x *EntityEvent_TagLocator) ProtoReflect() protoreflect.Message {
+ mi := &file_banyandb_database_v2_event_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EntityEvent_TagLocator.ProtoReflect.Descriptor instead.
+func (*EntityEvent_TagLocator) Descriptor() ([]byte, []int) {
+ return file_banyandb_database_v2_event_proto_rawDescGZIP(), []int{1, 0}
+}
+
+func (x *EntityEvent_TagLocator) GetFamilyOffset() uint32 {
+ if x != nil {
+ return x.FamilyOffset
+ }
+ return 0
+}
+
+func (x *EntityEvent_TagLocator) GetTagOffset() uint32 {
+ if x != nil {
+ return x.TagOffset
+ }
+ return 0
+}
+
var File_banyandb_database_v2_event_proto protoreflect.FileDescriptor
var file_banyandb_database_v2_event_proto_rawDesc = []byte{
@@ -249,34 +304,40 @@ var file_banyandb_database_v2_event_proto_rawDesc = []byte{
0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x69,
0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
- 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x22, 0xed, 0x01, 0x0a, 0x0b, 0x53,
- 0x65, 0x72, 0x69, 0x65, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x65,
- 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x62, 0x61, 0x6e,
- 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x32, 0x2e,
- 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x06, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73,
- 0x12, 0x42, 0x0a, 0x1e, 0x74, 0x61, 0x67, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x5f, 0x63,
- 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f,
- 0x69, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x1a, 0x74, 0x61, 0x67, 0x73, 0x4e, 0x61,
- 0x6d, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x65, 0x53, 0x65, 0x72, 0x69,
- 0x65, 0x73, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03,
- 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e,
- 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x76, 0x32, 0x2e, 0x41, 0x63, 0x74, 0x69,
- 0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x69,
- 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
- 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
- 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x2a, 0x43, 0x0a, 0x06, 0x41, 0x63,
- 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55,
- 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a,
- 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x55, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d,
- 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42,
- 0x72, 0x0a, 0x2a, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x73, 0x6b,
- 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64,
- 0x62, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x76, 0x32, 0x5a, 0x44, 0x67,
- 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65,
- 0x2f, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x61, 0x6e, 0x79,
- 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62,
- 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65,
- 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x22, 0xd2, 0x02, 0x0a, 0x0b, 0x45,
+ 0x6e, 0x74, 0x69, 0x74, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x36, 0x0a, 0x07, 0x73, 0x75,
+ 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x62, 0x61,
+ 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x32,
+ 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65,
+ 0x63, 0x74, 0x12, 0x53, 0x0a, 0x0e, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6c, 0x6f, 0x63,
+ 0x61, 0x74, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x62, 0x61, 0x6e,
+ 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x76,
+ 0x32, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x54, 0x61,
+ 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x0d, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79,
+ 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x34, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f,
+ 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e,
+ 0x64, 0x62, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x76, 0x32, 0x2e, 0x41,
+ 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a,
+ 0x04, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
+ 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
+ 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x1a, 0x50, 0x0a,
+ 0x0a, 0x54, 0x61, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x66,
+ 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x0d, 0x52, 0x0c, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74,
+ 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02,
+ 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x74, 0x61, 0x67, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x2a,
+ 0x43, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x43, 0x54,
+ 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10,
+ 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x55, 0x54, 0x10,
+ 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45,
+ 0x54, 0x45, 0x10, 0x02, 0x42, 0x72, 0x0a, 0x2a, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63,
+ 0x68, 0x65, 0x2e, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x62, 0x61,
+ 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x2e,
+ 0x76, 0x32, 0x5a, 0x44, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61,
+ 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67,
+ 0x2d, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72,
+ 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x64, 0x61, 0x74,
+ 0x61, 0x62, 0x61, 0x73, 0x65, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -292,27 +353,29 @@ func file_banyandb_database_v2_event_proto_rawDescGZIP() []byte {
}
var file_banyandb_database_v2_event_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_banyandb_database_v2_event_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_banyandb_database_v2_event_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_banyandb_database_v2_event_proto_goTypes = []interface{}{
- (Action)(0), // 0: banyandb.database.v2.Action
- (*ShardEvent)(nil), // 1: banyandb.database.v2.ShardEvent
- (*SeriesEvent)(nil), // 2: banyandb.database.v2.SeriesEvent
- (*Shard)(nil), // 3: banyandb.database.v2.Shard
- (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp
- (*v2.Metadata)(nil), // 5: banyandb.common.v2.Metadata
+ (Action)(0), // 0: banyandb.database.v2.Action
+ (*ShardEvent)(nil), // 1: banyandb.database.v2.ShardEvent
+ (*EntityEvent)(nil), // 2: banyandb.database.v2.EntityEvent
+ (*EntityEvent_TagLocator)(nil), // 3: banyandb.database.v2.EntityEvent.TagLocator
+ (*Shard)(nil), // 4: banyandb.database.v2.Shard
+ (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp
+ (*v2.Metadata)(nil), // 6: banyandb.common.v2.Metadata
}
var file_banyandb_database_v2_event_proto_depIdxs = []int32{
- 3, // 0: banyandb.database.v2.ShardEvent.shard:type_name -> banyandb.database.v2.Shard
+ 4, // 0: banyandb.database.v2.ShardEvent.shard:type_name -> banyandb.database.v2.Shard
0, // 1: banyandb.database.v2.ShardEvent.action:type_name -> banyandb.database.v2.Action
- 4, // 2: banyandb.database.v2.ShardEvent.time:type_name -> google.protobuf.Timestamp
- 5, // 3: banyandb.database.v2.SeriesEvent.series:type_name -> banyandb.common.v2.Metadata
- 0, // 4: banyandb.database.v2.SeriesEvent.action:type_name -> banyandb.database.v2.Action
- 4, // 5: banyandb.database.v2.SeriesEvent.time:type_name -> google.protobuf.Timestamp
- 6, // [6:6] is the sub-list for method output_type
- 6, // [6:6] is the sub-list for method input_type
- 6, // [6:6] is the sub-list for extension type_name
- 6, // [6:6] is the sub-list for extension extendee
- 0, // [0:6] is the sub-list for field type_name
+ 5, // 2: banyandb.database.v2.ShardEvent.time:type_name -> google.protobuf.Timestamp
+ 6, // 3: banyandb.database.v2.EntityEvent.subject:type_name -> banyandb.common.v2.Metadata
+ 3, // 4: banyandb.database.v2.EntityEvent.entity_locator:type_name -> banyandb.database.v2.EntityEvent.TagLocator
+ 0, // 5: banyandb.database.v2.EntityEvent.action:type_name -> banyandb.database.v2.Action
+ 5, // 6: banyandb.database.v2.EntityEvent.time:type_name -> google.protobuf.Timestamp
+ 7, // [7:7] is the sub-list for method output_type
+ 7, // [7:7] is the sub-list for method input_type
+ 7, // [7:7] is the sub-list for extension type_name
+ 7, // [7:7] is the sub-list for extension extendee
+ 0, // [0:7] is the sub-list for field type_name
}
func init() { file_banyandb_database_v2_event_proto_init() }
@@ -336,7 +399,19 @@ func file_banyandb_database_v2_event_proto_init() {
}
}
file_banyandb_database_v2_event_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*SeriesEvent); i {
+ switch v := v.(*EntityEvent); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_banyandb_database_v2_event_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*EntityEvent_TagLocator); i {
case 0:
return &v.state
case 1:
@@ -354,7 +429,7 @@ func file_banyandb_database_v2_event_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_banyandb_database_v2_event_proto_rawDesc,
NumEnums: 1,
- NumMessages: 2,
+ NumMessages: 3,
NumExtensions: 0,
NumServices: 0,
},
diff --git a/api/proto/banyandb/database/v2/event.proto b/api/proto/banyandb/database/v2/event.proto
index 530174a..bec7d9d 100644
--- a/api/proto/banyandb/database/v2/event.proto
+++ b/api/proto/banyandb/database/v2/event.proto
@@ -39,9 +39,13 @@ message ShardEvent {
google.protobuf.Timestamp time = 3;
}
-message SeriesEvent {
- common.v2.Metadata series = 1;
- repeated string tags_names_composite_series_id = 2;
+message EntityEvent {
+ common.v2.Metadata subject = 1;
+ message TagLocator {
+ uint32 family_offset = 1;
+ uint32 tag_offset = 2;
+ }
+ repeated TagLocator entity_locator = 2;
Action action = 3;
- google.protobuf.Timestamp time = 4;
+ google.protobuf.Timestamp time = 4;
}
diff --git a/api/proto/banyandb/model/v2/common.pb.go b/api/proto/banyandb/model/v2/common.pb.go
index f848ce7..18cdaea 100644
--- a/api/proto/banyandb/model/v2/common.pb.go
+++ b/api/proto/banyandb/model/v2/common.pb.go
@@ -363,6 +363,53 @@ func (*TagValue_IntArray) isTagValue_Value() {}
func (*TagValue_BinaryData) isTagValue_Value() {}
+type TagFamilyForWrite struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Tags []*TagValue `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty"`
+}
+
+func (x *TagFamilyForWrite) Reset() {
+ *x = TagFamilyForWrite{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_banyandb_model_v2_common_proto_msgTypes[5]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *TagFamilyForWrite) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TagFamilyForWrite) ProtoMessage() {}
+
+func (x *TagFamilyForWrite) ProtoReflect() protoreflect.Message {
+ mi := &file_banyandb_model_v2_common_proto_msgTypes[5]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use TagFamilyForWrite.ProtoReflect.Descriptor instead.
+func (*TagFamilyForWrite) Descriptor() ([]byte, []int) {
+ return file_banyandb_model_v2_common_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *TagFamilyForWrite) GetTags() []*TagValue {
+ if x != nil {
+ return x.Tags
+ }
+ return nil
+}
+
var File_banyandb_model_v2_common_proto protoreflect.FileDescriptor
var file_banyandb_model_v2_common_proto_rawDesc = []byte{
@@ -398,14 +445,19 @@ var file_banyandb_model_v2_common_proto_rawDesc = []byte{
0x61, 0x79, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, 0x21,
0x0a, 0x0b, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20,
0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0a, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74,
- 0x61, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x6c, 0x0a, 0x27, 0x6f, 0x72,
- 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b,
- 0x69, 0x6e, 0x67, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x6d, 0x6f, 0x64,
- 0x65, 0x6c, 0x2e, 0x76, 0x32, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
- 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b,
- 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69,
- 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f,
- 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x61, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x44, 0x0a, 0x11, 0x54, 0x61,
+ 0x67, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x46, 0x6f, 0x72, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12,
+ 0x2f, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e,
+ 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76,
+ 0x32, 0x2e, 0x54, 0x61, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73,
+ 0x42, 0x6c, 0x0a, 0x27, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x73,
+ 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e,
+ 0x64, 0x62, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x32, 0x5a, 0x41, 0x67, 0x69, 0x74,
+ 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x73,
+ 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e,
+ 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x61, 0x6e,
+ 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x76, 0x32, 0x62, 0x06,
+ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -420,26 +472,28 @@ func file_banyandb_model_v2_common_proto_rawDescGZIP() []byte {
return file_banyandb_model_v2_common_proto_rawDescData
}
-var file_banyandb_model_v2_common_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_banyandb_model_v2_common_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_banyandb_model_v2_common_proto_goTypes = []interface{}{
- (*Str)(nil), // 0: banyandb.model.v2.Str
- (*Int)(nil), // 1: banyandb.model.v2.Int
- (*StrArray)(nil), // 2: banyandb.model.v2.StrArray
- (*IntArray)(nil), // 3: banyandb.model.v2.IntArray
- (*TagValue)(nil), // 4: banyandb.model.v2.TagValue
- (structpb.NullValue)(0), // 5: google.protobuf.NullValue
+ (*Str)(nil), // 0: banyandb.model.v2.Str
+ (*Int)(nil), // 1: banyandb.model.v2.Int
+ (*StrArray)(nil), // 2: banyandb.model.v2.StrArray
+ (*IntArray)(nil), // 3: banyandb.model.v2.IntArray
+ (*TagValue)(nil), // 4: banyandb.model.v2.TagValue
+ (*TagFamilyForWrite)(nil), // 5: banyandb.model.v2.TagFamilyForWrite
+ (structpb.NullValue)(0), // 6: google.protobuf.NullValue
}
var file_banyandb_model_v2_common_proto_depIdxs = []int32{
- 5, // 0: banyandb.model.v2.TagValue.null:type_name -> google.protobuf.NullValue
+ 6, // 0: banyandb.model.v2.TagValue.null:type_name -> google.protobuf.NullValue
0, // 1: banyandb.model.v2.TagValue.str:type_name -> banyandb.model.v2.Str
2, // 2: banyandb.model.v2.TagValue.str_array:type_name -> banyandb.model.v2.StrArray
1, // 3: banyandb.model.v2.TagValue.int:type_name -> banyandb.model.v2.Int
3, // 4: banyandb.model.v2.TagValue.int_array:type_name -> banyandb.model.v2.IntArray
- 5, // [5:5] is the sub-list for method output_type
- 5, // [5:5] is the sub-list for method input_type
- 5, // [5:5] is the sub-list for extension type_name
- 5, // [5:5] is the sub-list for extension extendee
- 0, // [0:5] is the sub-list for field type_name
+ 4, // 5: banyandb.model.v2.TagFamilyForWrite.tags:type_name -> banyandb.model.v2.TagValue
+ 6, // [6:6] is the sub-list for method output_type
+ 6, // [6:6] is the sub-list for method input_type
+ 6, // [6:6] is the sub-list for extension type_name
+ 6, // [6:6] is the sub-list for extension extendee
+ 0, // [0:6] is the sub-list for field type_name
}
func init() { file_banyandb_model_v2_common_proto_init() }
@@ -508,6 +562,18 @@ func file_banyandb_model_v2_common_proto_init() {
return nil
}
}
+ file_banyandb_model_v2_common_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*TagFamilyForWrite); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
}
file_banyandb_model_v2_common_proto_msgTypes[4].OneofWrappers = []interface{}{
(*TagValue_Null)(nil),
@@ -523,7 +589,7 @@ func file_banyandb_model_v2_common_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_banyandb_model_v2_common_proto_rawDesc,
NumEnums: 0,
- NumMessages: 5,
+ NumMessages: 6,
NumExtensions: 0,
NumServices: 0,
},
diff --git a/api/proto/banyandb/model/v2/common.proto b/api/proto/banyandb/model/v2/common.proto
index e9c51a5..5738520 100644
--- a/api/proto/banyandb/model/v2/common.proto
+++ b/api/proto/banyandb/model/v2/common.proto
@@ -50,3 +50,7 @@ message TagValue {
bytes binary_data = 6;
}
}
+
+message TagFamilyForWrite {
+ repeated TagValue tags = 1;
+}
diff --git a/api/proto/banyandb/stream/v2/write.pb.go b/api/proto/banyandb/stream/v2/write.pb.go
index 608436d..c062805 100644
--- a/api/proto/banyandb/stream/v2/write.pb.go
+++ b/api/proto/banyandb/stream/v2/write.pb.go
@@ -31,8 +31,8 @@ import (
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
- v2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
- v21 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
+ v21 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
+ v2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
)
const (
@@ -54,7 +54,7 @@ type ElementValue struct {
// 2) or the timestamp of a log
Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
// the order of tag_families' items match the stream schema
- TagFamilies []*ElementValue_TagFamily `protobuf:"bytes,3,rep,name=tag_families,json=tagFamilies,proto3" json:"tag_families,omitempty"`
+ TagFamilies []*v2.TagFamilyForWrite `protobuf:"bytes,3,rep,name=tag_families,json=tagFamilies,proto3" json:"tag_families,omitempty"`
}
func (x *ElementValue) Reset() {
@@ -103,7 +103,7 @@ func (x *ElementValue) GetTimestamp() *timestamppb.Timestamp {
return nil
}
-func (x *ElementValue) GetTagFamilies() []*ElementValue_TagFamily {
+func (x *ElementValue) GetTagFamilies() []*v2.TagFamilyForWrite {
if x != nil {
return x.TagFamilies
}
@@ -116,7 +116,7 @@ type WriteRequest struct {
unknownFields protoimpl.UnknownFields
// the metadata is only required in the first write.
- Metadata *v2.Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
+ Metadata *v21.Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
// the element is required.
Element *ElementValue `protobuf:"bytes,2,opt,name=element,proto3" json:"element,omitempty"`
}
@@ -153,7 +153,7 @@ func (*WriteRequest) Descriptor() ([]byte, []int) {
return file_banyandb_stream_v2_write_proto_rawDescGZIP(), []int{1}
}
-func (x *WriteRequest) GetMetadata() *v2.Metadata {
+func (x *WriteRequest) GetMetadata() *v21.Metadata {
if x != nil {
return x.Metadata
}
@@ -205,16 +205,18 @@ func (*WriteResponse) Descriptor() ([]byte, []int) {
return file_banyandb_stream_v2_write_proto_rawDescGZIP(), []int{2}
}
-type ElementValue_TagFamily struct {
+type InternalWriteRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Tags []*v21.TagValue `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty"`
+ ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
+ SeriesHash []byte `protobuf:"bytes,2,opt,name=series_hash,json=seriesHash,proto3" json:"series_hash,omitempty"`
+ Request *WriteRequest `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"`
}
-func (x *ElementValue_TagFamily) Reset() {
- *x = ElementValue_TagFamily{}
+func (x *InternalWriteRequest) Reset() {
+ *x = InternalWriteRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_banyandb_stream_v2_write_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -222,13 +224,13 @@ func (x *ElementValue_TagFamily) Reset() {
}
}
-func (x *ElementValue_TagFamily) String() string {
+func (x *InternalWriteRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*ElementValue_TagFamily) ProtoMessage() {}
+func (*InternalWriteRequest) ProtoMessage() {}
-func (x *ElementValue_TagFamily) ProtoReflect() protoreflect.Message {
+func (x *InternalWriteRequest) ProtoReflect() protoreflect.Message {
mi := &file_banyandb_stream_v2_write_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -240,14 +242,28 @@ func (x *ElementValue_TagFamily) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use ElementValue_TagFamily.ProtoReflect.Descriptor instead.
-func (*ElementValue_TagFamily) Descriptor() ([]byte, []int) {
- return file_banyandb_stream_v2_write_proto_rawDescGZIP(), []int{0, 0}
+// Deprecated: Use InternalWriteRequest.ProtoReflect.Descriptor instead.
+func (*InternalWriteRequest) Descriptor() ([]byte, []int) {
+ return file_banyandb_stream_v2_write_proto_rawDescGZIP(), []int{3}
}
-func (x *ElementValue_TagFamily) GetTags() []*v21.TagValue {
+func (x *InternalWriteRequest) GetShardId() uint32 {
if x != nil {
- return x.Tags
+ return x.ShardId
+ }
+ return 0
+}
+
+func (x *InternalWriteRequest) GetSeriesHash() []byte {
+ if x != nil {
+ return x.SeriesHash
+ }
+ return nil
+}
+
+func (x *InternalWriteRequest) GetRequest() *WriteRequest {
+ if x != nil {
+ return x.Request
}
return nil
}
@@ -264,39 +280,44 @@ var file_banyandb_stream_v2_write_proto_rawDesc = []byte{
0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62,
0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
- 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf4, 0x01, 0x0a, 0x0c, 0x45, 0x6c, 0x65, 0x6d, 0x65,
+ 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb0, 0x01, 0x0a, 0x0c, 0x45, 0x6c, 0x65, 0x6d, 0x65,
0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6c, 0x65, 0x6d, 0x65,
0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x6c, 0x65,
0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
- 0x12, 0x4d, 0x0a, 0x0c, 0x74, 0x61, 0x67, 0x5f, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73,
- 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64,
- 0x62, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x6c, 0x65, 0x6d,
- 0x65, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x46, 0x61, 0x6d, 0x69,
- 0x6c, 0x79, 0x52, 0x0b, 0x74, 0x61, 0x67, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73, 0x1a,
- 0x3c, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x2f, 0x0a, 0x04,
- 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x62, 0x61, 0x6e,
- 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x32, 0x2e, 0x54,
- 0x61, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x84, 0x01,
- 0x0a, 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38,
- 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
- 0x32, 0x1c, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x6d,
- 0x6f, 0x6e, 0x2e, 0x76, 0x32, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08,
- 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3a, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d,
- 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x62, 0x61, 0x6e, 0x79,
- 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x32, 0x2e, 0x45,
- 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x65, 0x6c, 0x65,
- 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x0f, 0x0a, 0x0d, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73,
- 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x6e, 0x0a, 0x28, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61,
- 0x63, 0x68, 0x65, 0x2e, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x62,
- 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76,
- 0x32, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70,
- 0x61, 0x63, 0x68, 0x65, 0x2f, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2d,
- 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f,
- 0x74, 0x6f, 0x2f, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x73, 0x74, 0x72, 0x65,
- 0x61, 0x6d, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x12, 0x47, 0x0a, 0x0c, 0x74, 0x61, 0x67, 0x5f, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73,
+ 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64,
+ 0x62, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x76, 0x32, 0x2e, 0x54, 0x61, 0x67, 0x46, 0x61,
+ 0x6d, 0x69, 0x6c, 0x79, 0x46, 0x6f, 0x72, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x0b, 0x74, 0x61,
+ 0x67, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x0c, 0x57, 0x72,
+ 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65,
+ 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x62,
+ 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76,
+ 0x32, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61,
+ 0x64, 0x61, 0x74, 0x61, 0x12, 0x3a, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18,
+ 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62,
+ 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65,
+ 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74,
+ 0x22, 0x0f, 0x0a, 0x0d, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+ 0x65, 0x22, 0x8e, 0x01, 0x0a, 0x14, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x57, 0x72,
+ 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68,
+ 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68,
+ 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f,
+ 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x69,
+ 0x65, 0x73, 0x48, 0x61, 0x73, 0x68, 0x12, 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73,
+ 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e,
+ 0x64, 0x62, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x72, 0x69,
+ 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65,
+ 0x73, 0x74, 0x42, 0x6e, 0x0a, 0x28, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65,
+ 0x2e, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x62, 0x61, 0x6e, 0x79,
+ 0x61, 0x6e, 0x64, 0x62, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x32, 0x5a, 0x42,
+ 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68,
+ 0x65, 0x2f, 0x73, 0x6b, 0x79, 0x77, 0x61, 0x6c, 0x6b, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x61, 0x6e,
+ 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f,
+ 0x62, 0x61, 0x6e, 0x79, 0x61, 0x6e, 0x64, 0x62, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f,
+ 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -313,20 +334,20 @@ func file_banyandb_stream_v2_write_proto_rawDescGZIP() []byte {
var file_banyandb_stream_v2_write_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_banyandb_stream_v2_write_proto_goTypes = []interface{}{
- (*ElementValue)(nil), // 0: banyandb.stream.v2.ElementValue
- (*WriteRequest)(nil), // 1: banyandb.stream.v2.WriteRequest
- (*WriteResponse)(nil), // 2: banyandb.stream.v2.WriteResponse
- (*ElementValue_TagFamily)(nil), // 3: banyandb.stream.v2.ElementValue.TagFamily
- (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp
- (*v2.Metadata)(nil), // 5: banyandb.common.v2.Metadata
- (*v21.TagValue)(nil), // 6: banyandb.model.v2.TagValue
+ (*ElementValue)(nil), // 0: banyandb.stream.v2.ElementValue
+ (*WriteRequest)(nil), // 1: banyandb.stream.v2.WriteRequest
+ (*WriteResponse)(nil), // 2: banyandb.stream.v2.WriteResponse
+ (*InternalWriteRequest)(nil), // 3: banyandb.stream.v2.InternalWriteRequest
+ (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp
+ (*v2.TagFamilyForWrite)(nil), // 5: banyandb.model.v2.TagFamilyForWrite
+ (*v21.Metadata)(nil), // 6: banyandb.common.v2.Metadata
}
var file_banyandb_stream_v2_write_proto_depIdxs = []int32{
4, // 0: banyandb.stream.v2.ElementValue.timestamp:type_name -> google.protobuf.Timestamp
- 3, // 1: banyandb.stream.v2.ElementValue.tag_families:type_name -> banyandb.stream.v2.ElementValue.TagFamily
- 5, // 2: banyandb.stream.v2.WriteRequest.metadata:type_name -> banyandb.common.v2.Metadata
+ 5, // 1: banyandb.stream.v2.ElementValue.tag_families:type_name -> banyandb.model.v2.TagFamilyForWrite
+ 6, // 2: banyandb.stream.v2.WriteRequest.metadata:type_name -> banyandb.common.v2.Metadata
0, // 3: banyandb.stream.v2.WriteRequest.element:type_name -> banyandb.stream.v2.ElementValue
- 6, // 4: banyandb.stream.v2.ElementValue.TagFamily.tags:type_name -> banyandb.model.v2.TagValue
+ 1, // 4: banyandb.stream.v2.InternalWriteRequest.request:type_name -> banyandb.stream.v2.WriteRequest
5, // [5:5] is the sub-list for method output_type
5, // [5:5] is the sub-list for method input_type
5, // [5:5] is the sub-list for extension type_name
@@ -377,7 +398,7 @@ func file_banyandb_stream_v2_write_proto_init() {
}
}
file_banyandb_stream_v2_write_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ElementValue_TagFamily); i {
+ switch v := v.(*InternalWriteRequest); i {
case 0:
return &v.state
case 1:
diff --git a/api/proto/banyandb/stream/v2/write.proto b/api/proto/banyandb/stream/v2/write.proto
index c845dbc..f671a6c 100644
--- a/api/proto/banyandb/stream/v2/write.proto
+++ b/api/proto/banyandb/stream/v2/write.proto
@@ -33,11 +33,8 @@ message ElementValue {
// 1) either the start time of a Span/Segment,
// 2) or the timestamp of a log
google.protobuf.Timestamp timestamp = 2;
- message TagFamily {
- repeated model.v2.TagValue tags = 1;
- }
// the order of tag_families' items match the stream schema
- repeated TagFamily tag_families = 3;
+ repeated model.v2.TagFamilyForWrite tag_families = 3;
}
message WriteRequest {
@@ -48,3 +45,9 @@ message WriteRequest {
}
message WriteResponse {}
+
+message InternalWriteRequest {
+ uint32 shard_id = 1;
+ bytes series_hash = 2;
+ WriteRequest request = 3;
+}
diff --git a/api/schema/index.go b/api/schema/index.go
deleted file mode 100644
index adaa8ca..0000000
--- a/api/schema/index.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package schema
-
-import (
- "github.com/apache/skywalking-banyandb/api/common"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
-)
-
-var IndexRuleKindVersion = common.KindVersion{Version: "v1", Kind: "schema-index-rule"}
-var IndexRuleBindingKindVersion = common.KindVersion{Version: "v1", Kind: "schema-index-rule-binding"}
-
-type IndexRule struct {
- common.KindVersion
- Spec *v1.IndexRule
-}
-
-type IndexRuleBinding struct {
- common.KindVersion
- Spec *v1.IndexRuleBinding
-}
diff --git a/api/schema/series.go b/api/schema/series.go
deleted file mode 100644
index e75a4cf..0000000
--- a/api/schema/series.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package schema
-
-import (
- "github.com/apache/skywalking-banyandb/api/common"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
-)
-
-var SeriesKindVersion = common.KindVersion{Version: "v1", Kind: "schema-series"}
-
-type TraceSeries struct {
- common.KindVersion
- Spec *v1.TraceSeries
-}
diff --git a/banyand/index/index.go b/banyand/index/index.go
deleted file mode 100644
index 262c665..0000000
--- a/banyand/index/index.go
+++ /dev/null
@@ -1,289 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package index
-
-import (
- "context"
- "sync"
-
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/event"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-var (
- ErrShardNotFound = errors.New("series doesn't exist")
- ErrTraceSeriesNotFound = errors.New("trace series not found")
-)
-
-type Condition struct {
- Key string
- Values [][]byte
- Op modelv1.PairQuery_BinaryOp
-}
-
-type Field struct {
- ChunkID common.ChunkID
- Name string
- Value []byte
-}
-
-//go:generate mockgen -destination=./index_mock.go -package=index . Service
-type Repo interface {
- Search(seriesMeta common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error)
- Insert(seriesMeta common.Metadata, shardID uint, fields *Field) error
-}
-
-type Builder interface {
- run.PreRunner
- run.Service
-}
-
-type ReadyOption func(map[string]*series) bool
-
-func MetaExists(group, name string) ReadyOption {
- seriesID := &commonv1.Metadata{
- Name: "sw",
- Group: "default",
- }
- return func(m map[string]*series) bool {
- if _, ok := m[compositeSeriesID(seriesID)]; ok {
- return true
- }
- return false
- }
-}
-
-type Service interface {
- Repo
- Builder
- Ready(context.Context, ...ReadyOption) bool
-}
-
-type series struct {
- repo map[uint]*shard
-}
-
-type shard struct {
- meta map[string][]*databasev1.IndexObject
- store tsdb.GlobalStore
-}
-
-type service struct {
- meta *indexMeta
- log *logger.Logger
- repo discovery.ServiceRepo
- stopCh chan struct{}
- indexRuleListener *indexRuleListener
-}
-
-func NewService(_ context.Context, repo discovery.ServiceRepo) (Service, error) {
- svc := &service{
- repo: repo,
- indexRuleListener: &indexRuleListener{},
- }
- svc.meta = &indexMeta{
- meta: make(map[string]*series),
- }
- svc.indexRuleListener.indexMeta = svc.meta
- svc.indexRuleListener.closeFunc = func() {
- svc.stopCh <- struct{}{}
- }
- return svc, nil
-}
-
-func (s *service) Insert(series common.Metadata, shardID uint, field *Field) error {
- sd, err := s.getShard(series, shardID)
- if err != nil {
- return err
- }
- objects, ok := sd.meta[field.Name]
- if !ok {
- s.log.Debug().Str("field", field.Name).Msg("field is not indexed")
- return nil
- }
- for _, object := range objects {
- err = multierr.Append(err, sd.store.Insert(&tsdb.Field{
- Name: []byte(compositeFieldID(object.GetName(), field.Name)),
- Value: field.Value,
- }, field.ChunkID))
- }
- return err
-}
-
-func (s *service) getShard(series common.Metadata, shardID uint) (*shard, error) {
- ss := s.meta.get(series.Spec)
- if ss == nil {
- return nil, errors.Wrapf(ErrTraceSeriesNotFound, "identify:%s", compositeSeriesID(series.Spec))
- }
- sd, existSearcher := ss.repo[shardID]
- if !existSearcher {
- return nil, errors.Wrapf(ErrShardNotFound, "shardID:%d", shardID)
- }
- return sd, nil
-}
-
-func (s *service) Name() string {
- return "index"
-}
-
-func (s *service) PreRun() error {
- //TODO: listen to written data
- s.log = logger.GetLogger("index")
- s.indexRuleListener.log = s.log
- return s.repo.Subscribe(event.TopicIndexRule, s.indexRuleListener)
-}
-
-func (s *service) Serve() error {
- s.stopCh = make(chan struct{})
- <-s.stopCh
- return nil
-}
-
-func (s *service) GracefulStop() {
- if s.stopCh != nil {
- close(s.stopCh)
- }
-}
-
-func (s *service) Ready(ctx context.Context, options ...ReadyOption) bool {
- options = append(options, func(m map[string]*series) bool {
- return len(m) > 0
- })
-
- for {
- select {
- case <-ctx.Done():
- return ctx.Err() == nil
- default:
- allMatches := true
- for _, opt := range options {
- if allMatches = opt(s.meta.meta); !allMatches {
- break
- }
- }
- if !allMatches {
- continue
- }
- return true
- }
- }
-}
-
-type indexMeta struct {
- meta map[string]*series
- sync.RWMutex
-}
-
-func (i *indexMeta) get(series *commonv1.Metadata) *series {
- i.RWMutex.RLock()
- defer i.RWMutex.RUnlock()
- s, ok := i.meta[compositeSeriesID(series)]
- if ok {
- return s
- }
- return nil
-}
-
-type indexRuleListener struct {
- log *logger.Logger
- indexMeta *indexMeta
- closeFunc func()
-}
-
-func (i *indexRuleListener) Rev(message bus.Message) (resp bus.Message) {
- indexRuleEvent, ok := message.Data().(*databasev1.IndexRuleEvent)
- if !ok {
- i.log.Warn().Msg("invalid event data type")
- return
- }
- i.log.Info().
- Str("action", databasev1.Action_name[int32(indexRuleEvent.Action)]).
- Str("series-name", indexRuleEvent.Series.Name).
- Str("series-group", indexRuleEvent.Series.Group).
- Msg("received an index rule")
- i.indexMeta.Lock()
- defer i.indexMeta.Unlock()
- switch indexRuleEvent.Action {
- case databasev1.Action_ACTION_PUT:
- seriesID := compositeSeriesID(indexRuleEvent.Series)
- newSeries := &series{
- repo: make(map[uint]*shard),
- }
- for _, rule := range indexRuleEvent.Rules {
- store := tsdb.NewStore(indexRuleEvent.Series.Name, indexRuleEvent.Series.Group, uint(rule.ShardId))
- fields := make([]tsdb.FieldSpec, 0, len(rule.Rules))
- meta := make(map[string][]*databasev1.IndexObject)
- for _, indexRule := range rule.GetRules() {
- for _, object := range indexRule.Objects {
- fieldsSize := len(object.Fields)
- if fieldsSize > 1 {
- //TODO: to support composited index
- i.log.Error().Str("name", object.Name).
- Msg("index module doesn't support composited index object")
- i.closeFunc()
- } else if fieldsSize < 1 {
- continue
- }
- field := object.Fields[0]
- fieldSpec := tsdb.FieldSpec{
- Name: compositeFieldID(object.Name, field),
- }
- fields = append(fields, fieldSpec)
- objects, existed := meta[field]
- if !existed {
- objects = make([]*databasev1.IndexObject, 0, 1)
- }
- objects = append(objects, object)
- meta[field] = objects
- }
- }
- err := store.Initialize(fields)
- if err != nil {
- i.log.Warn().Err(err).Msg("failed to initialize index getShard")
- }
- newSeries.repo[uint(rule.ShardId)] = &shard{
- store: store,
- meta: meta,
- }
- }
- i.indexMeta.meta[seriesID] = newSeries
- default:
- i.log.Warn().Msg("unsupported action")
- }
- return
-}
-
-func compositeFieldID(indexObjectName, field string) string {
- return indexObjectName + ":" + field
-}
-
-func compositeSeriesID(series *commonv1.Metadata) string {
- return series.Name + "-" + series.Group
-}
diff --git a/banyand/index/index_test.go b/banyand/index/index_test.go
deleted file mode 100644
index 7730f91..0000000
--- a/banyand/index/index_test.go
+++ /dev/null
@@ -1,181 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package index
-
-import (
- "context"
- "math"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "google.golang.org/protobuf/types/known/timestamppb"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/event"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-func Test_service_Insert(t *testing.T) {
- tester := assert.New(t)
- type args struct {
- series common.Metadata
- shardID uint
- field *Field
- }
- tests := []struct {
- name string
- args args
- wantErr bool
- }{
- {
- name: "str field",
- args: args{
- series: *common.NewMetadataByNameAndGroup("sw", "default"),
- shardID: 0,
- field: &Field{
- ChunkID: common.ChunkID(1),
- Name: "endpoint",
- Value: []byte("/test"),
- },
- },
- },
- {
- name: "int field",
- args: args{
- series: *common.NewMetadataByNameAndGroup("sw", "default"),
- shardID: 1,
- field: &Field{
- ChunkID: common.ChunkID(2),
- Name: "duration",
- Value: convert.Int64ToBytes(500),
- },
- },
- },
- {
- name: "unknown series",
- args: args{
- series: *common.NewMetadataByNameAndGroup("unknown", "default"),
- shardID: 0,
- field: &Field{
- ChunkID: common.ChunkID(2),
- Name: "duration",
- Value: convert.Int64ToBytes(500),
- },
- },
- wantErr: true,
- },
- {
- name: "unknown shard",
- args: args{
- series: *common.NewMetadataByNameAndGroup("sw", "default"),
- shardID: math.MaxInt64,
- field: &Field{
- ChunkID: common.ChunkID(2),
- Name: "duration",
- Value: convert.Int64ToBytes(500),
- },
- },
- wantErr: true,
- },
- {
- name: "unknown field",
- args: args{
- series: *common.NewMetadataByNameAndGroup("sw", "default"),
- shardID: 0,
- field: &Field{
- ChunkID: common.ChunkID(2),
- Name: "unknown",
- Value: convert.Int64ToBytes(500),
- },
- },
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := setUpModules(tester)
- if err := s.Insert(tt.args.series, tt.args.shardID, tt.args.field); (err != nil) != tt.wantErr {
- t.Errorf("Write() error = %v, wantErr %v", err, tt.wantErr)
- }
- })
- }
-}
-
-func Test_service_Init(t *testing.T) {
- tester := assert.New(t)
- s := setUpModules(tester)
- tester.Equal(1, len(s.meta.meta))
- tester.Equal(2, len(s.meta.meta["sw-default"].repo))
-}
-
-func setUpModules(tester *assert.Assertions) *service {
- _ = logger.Bootstrap()
- repo, err := discovery.NewServiceRepo(context.TODO())
- tester.NoError(err)
- svc, err := NewService(context.TODO(), repo)
- tester.NoError(err)
- tester.NoError(svc.PreRun())
-
- rules := []*databasev1.IndexRule{
- {
- Objects: []*databasev1.IndexObject{
- {
- Name: "endpoint",
- Fields: []string{"endpoint"},
- },
- {
- Name: "duration",
- Fields: []string{"duration"},
- },
- },
- },
- }
- seriesID := &commonv1.Metadata{
- Name: "sw",
- Group: "default",
- }
- _, err = repo.Publish(event.TopicIndexRule, bus.NewMessage(bus.MessageID(time.Now().UnixNano()), &databasev1.IndexRuleEvent{
- Series: seriesID,
- Rules: []*databasev1.IndexRuleEvent_ShardedIndexRule{
- {
- ShardId: 0,
- Rules: rules,
- },
- {
- ShardId: 1,
- Rules: rules,
- },
- },
- Action: databasev1.Action_ACTION_PUT,
- Time: timestamppb.Now(),
- }))
- tester.NoError(err)
- s, ok := svc.(*service)
- tester.True(ok)
-
- ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancelFunc()
- tester.True(svc.Ready(ctx, MetaExists("default", "sw")))
- return s
-}
diff --git a/banyand/index/search.go b/banyand/index/search.go
deleted file mode 100644
index d25ad88..0000000
--- a/banyand/index/search.go
+++ /dev/null
@@ -1,356 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package index
-
-import (
- "encoding/base64"
- "encoding/json"
- "strings"
-
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
- "github.com/apache/skywalking-banyandb/pkg/bytes"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-var ErrNotRangeOperation = errors.New("this is not an range operation")
-
-type executable interface {
- execute() (posting.List, error)
-}
-
-type searchTree interface {
- executable
-}
-
-func (s *service) Search(series common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error) {
- sd, err := s.getShard(series, shardID)
- if err != nil {
- return nil, err
- }
- store := sd.store
- searcher, hasData := store.Window(startTime, endTime)
- if !hasData {
- return roaring.EmptyPostingList, nil
- }
- tree, errBuild := buildSearchTree(searcher, indexObjectName, conditions)
- if errBuild != nil {
- return nil, err
- }
- s.log.Debug().Interface("search-tree", tree).Msg("build search tree")
-
- result, err := tree.execute()
- if result == nil {
- return roaring.EmptyPostingList, err
- }
- return result, err
-}
-
-func buildSearchTree(searcher tsdb.Searcher, indexObject string, conditions []Condition) (searchTree, error) {
- condMap := toMap(indexObject, conditions)
- root := &andNode{
- node: &node{
- SubNodes: make([]executable, 0),
- searcher: searcher,
- },
- }
- for key, conds := range condMap {
- var rangeLeaf *rangeOp
- for _, cond := range conds {
- if rangeLeaf != nil && !rangeOP(cond.Op) {
- return nil, errors.Wrapf(ErrNotRangeOperation, "op:%s", cond.Op.String())
- }
- if rangeOP(cond.Op) {
- if rangeLeaf == nil {
- rangeLeaf = root.addRangeLeaf(key)
- }
- opts := rangeLeaf.Opts
- switch cond.Op {
- case modelv1.PairQuery_BINARY_OP_GT:
- opts.Lower = bytes.Join(cond.Values...)
- case modelv1.PairQuery_BINARY_OP_GE:
- opts.Lower = bytes.Join(cond.Values...)
- opts.IncludesLower = true
- case modelv1.PairQuery_BINARY_OP_LT:
- opts.Upper = bytes.Join(cond.Values...)
- case modelv1.PairQuery_BINARY_OP_LE:
- opts.Upper = bytes.Join(cond.Values...)
- opts.IncludesUpper = true
- }
- continue
- }
- switch cond.Op {
- case modelv1.PairQuery_BINARY_OP_EQ:
- root.addEq(key, cond.Values)
- case modelv1.PairQuery_BINARY_OP_NE:
- root.addNot(key, root.newEq(key, cond.Values))
- case modelv1.PairQuery_BINARY_OP_HAVING:
- n := root.addOrNode(len(cond.Values))
- for _, v := range cond.Values {
- n.addEq(key, [][]byte{v})
- }
- case modelv1.PairQuery_BINARY_OP_NOT_HAVING:
- n := root.newOrNode(len(cond.Values))
- for _, v := range cond.Values {
- n.addEq(key, [][]byte{v})
- }
- root.addNot(key, n)
- }
- }
- }
- return root, nil
-}
-
-func rangeOP(op modelv1.PairQuery_BinaryOp) bool {
- switch op {
- case modelv1.PairQuery_BINARY_OP_GT,
- modelv1.PairQuery_BINARY_OP_GE,
- modelv1.PairQuery_BINARY_OP_LT,
- modelv1.PairQuery_BINARY_OP_LE:
- return true
- }
- return false
-}
-
-func toMap(indexObject string, condition []Condition) map[string][]Condition {
- result := make(map[string][]Condition)
- for _, c := range condition {
- key := compositeFieldID(indexObject, c.Key)
- l, ok := result[key]
- if ok {
- l = append(l, c)
- result[key] = l
- continue
- }
- result[key] = []Condition{c}
- }
- return result
-}
-
-type logicalOP interface {
- executable
- merge(posting.List) error
-}
-
-type node struct {
- searcher tsdb.Searcher
- value posting.List
- SubNodes []executable `json:"sub_nodes,omitempty"`
-}
-
-func (n *node) newEq(key string, values [][]byte) *eq {
- return &eq{
- leaf: &leaf{
- Key: key,
- Values: values,
- searcher: n.searcher,
- },
- }
-}
-
-func (n *node) addEq(key string, values [][]byte) {
- n.SubNodes = append(n.SubNodes, n.newEq(key, values))
-}
-
-func (n *node) addNot(key string, inner executable) {
- n.SubNodes = append(n.SubNodes, ¬{
- Key: key,
- searcher: n.searcher,
- Inner: inner,
- })
-}
-
-func (n *node) addRangeLeaf(key string) *rangeOp {
- r := &rangeOp{
- leaf: &leaf{
- Key: key,
- searcher: n.searcher,
- },
- Opts: &tsdb.RangeOpts{},
- }
- n.SubNodes = append(n.SubNodes, r)
- return r
-}
-
-func (n *node) newOrNode(size int) *orNode {
- return &orNode{
- node: &node{
- searcher: n.searcher,
- SubNodes: make([]executable, 0, size),
- },
- }
-}
-
-func (n *node) addOrNode(size int) *orNode {
- on := n.newOrNode(size)
- n.SubNodes = append(n.SubNodes, on)
- return on
-}
-
-func (n *node) pop() (executable, bool) {
- if len(n.SubNodes) < 1 {
- return nil, false
- }
- sn := n.SubNodes[0]
- n.SubNodes = n.SubNodes[1:]
- return sn, true
-}
-
-func execute(n *node, lp logicalOP) (posting.List, error) {
- ex, hasNext := n.pop()
- if !hasNext {
- return n.value, nil
- }
- r, err := ex.execute()
- if err != nil {
- return nil, err
- }
- if n.value == nil {
- n.value = r
- return lp.execute()
- }
- err = lp.merge(r)
- if err != nil {
- return nil, err
- }
- if n.value.IsEmpty() {
- return n.value, nil
- }
- return lp.execute()
-}
-
-type andNode struct {
- *node
-}
-
-func (an *andNode) merge(list posting.List) error {
- return an.value.Intersect(list)
-}
-
-func (an *andNode) execute() (posting.List, error) {
- return execute(an.node, an)
-}
-
-func (an *andNode) MarshalJSON() ([]byte, error) {
- data := make(map[string]interface{}, 1)
- data["and"] = an.node.SubNodes
- return json.Marshal(data)
-}
-
-type orNode struct {
- *node
-}
-
-func (on *orNode) merge(list posting.List) error {
- return on.value.Union(list)
-}
-
-func (on *orNode) execute() (posting.List, error) {
- return execute(on.node, on)
-}
-
-func (on *orNode) MarshalJSON() ([]byte, error) {
- data := make(map[string]interface{}, 1)
- data["or"] = on.node.SubNodes
- return json.Marshal(data)
-}
-
-type leaf struct {
- executable
- Key string
- Values [][]byte
- searcher tsdb.Searcher
-}
-
-type not struct {
- executable
- Key string
- searcher tsdb.Searcher
- Inner executable
-}
-
-func (n *not) execute() (posting.List, error) {
- all := n.searcher.MatchField([]byte(n.Key))
- list, err := n.Inner.execute()
- if err != nil {
- return nil, err
- }
- err = all.Difference(list)
- return all, err
-}
-
-func (n *not) MarshalJSON() ([]byte, error) {
- data := make(map[string]interface{}, 1)
- data["not"] = n.Inner
- return json.Marshal(data)
-}
-
-type eq struct {
- *leaf
-}
-
-func (eq *eq) execute() (posting.List, error) {
- return eq.searcher.MatchTerms(&tsdb.Field{
- Name: []byte(eq.Key),
- Value: bytes.Join(eq.Values...),
- }), nil
-}
-
-func (eq *eq) MarshalJSON() ([]byte, error) {
- data := make(map[string]interface{}, 1)
- data["eq"] = eq.leaf
- return json.Marshal(data)
-}
-
-type rangeOp struct {
- *leaf
- Opts *tsdb.RangeOpts
-}
-
-func (r *rangeOp) execute() (posting.List, error) {
- return r.searcher.Range([]byte(r.Key), r.Opts), nil
-}
-
-func (r *rangeOp) MarshalJSON() ([]byte, error) {
- data := make(map[string]interface{}, 1)
- var builder strings.Builder
- if r.Opts.Lower != nil {
- if r.Opts.IncludesLower {
- builder.WriteString("[")
- } else {
- builder.WriteString("(")
- }
- }
- builder.WriteString(base64.StdEncoding.EncodeToString(r.Opts.Lower))
- builder.WriteString(",")
- builder.WriteString(base64.StdEncoding.EncodeToString(r.Opts.Upper))
- if r.Opts.Upper != nil {
- if r.Opts.IncludesUpper {
- builder.WriteString("]")
- } else {
- builder.WriteString(")")
- }
- }
- data["key"] = r.Key
- data["range"] = builder.String()
- return json.Marshal(data)
-}
diff --git a/banyand/index/search_test.go b/banyand/index/search_test.go
deleted file mode 100644
index 8423cec..0000000
--- a/banyand/index/search_test.go
+++ /dev/null
@@ -1,255 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package index
-
-import (
- "math"
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/skywalking-banyandb/api/common"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-func Test_service_Search(t *testing.T) {
- tester := assert.New(t)
- type args struct {
- indexObjectName string
- conditions []Condition
- }
- tests := []struct {
- name string
- args args
- want posting.List
- wantErr bool
- }{
- {
- name: "str equal",
- args: args{
- indexObjectName: "endpoint",
- conditions: []Condition{
- {
- Key: "endpoint",
- Op: modelv1.PairQuery_BINARY_OP_EQ,
- Values: [][]byte{[]byte("/product")},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(1),
- },
- {
- name: "str not equal",
- args: args{
- indexObjectName: "endpoint",
- conditions: []Condition{
- {
- Key: "endpoint",
- Op: modelv1.PairQuery_BINARY_OP_NE,
- Values: [][]byte{[]byte("/product")},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(2, 3),
- },
- {
- name: "str having",
- args: args{
- indexObjectName: "endpoint",
- conditions: []Condition{
- {
- Key: "endpoint",
- Op: modelv1.PairQuery_BINARY_OP_HAVING,
- Values: [][]byte{[]byte("/product"), []byte("/sales")},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(1, 3),
- },
- {
- name: "str not having",
- args: args{
- indexObjectName: "endpoint",
- conditions: []Condition{
- {
- Key: "endpoint",
- Op: modelv1.PairQuery_BINARY_OP_NOT_HAVING,
- Values: [][]byte{[]byte("/product"), []byte("/sales")},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(2),
- },
- {
- name: "int equal",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_EQ,
- Values: [][]byte{convert.Int64ToBytes(500)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(12),
- },
- {
- name: "int not equal",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_NE,
- Values: [][]byte{convert.Int64ToBytes(500)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(11, 13, 14),
- },
- {
- name: "int having",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_HAVING,
- Values: [][]byte{convert.Int64ToBytes(500), convert.Int64ToBytes(50)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(11, 12),
- },
- {
- name: "int not having",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_NOT_HAVING,
- Values: [][]byte{convert.Int64ToBytes(500), convert.Int64ToBytes(50)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(13, 14),
- },
- {
- name: "int in range",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_GT,
- Values: [][]byte{convert.Int64ToBytes(50)},
- },
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_LT,
- Values: [][]byte{convert.Int64ToBytes(5000)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(13, 12),
- },
- {
- name: "int includes edges",
- args: args{
- indexObjectName: "duration",
- conditions: []Condition{
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_GE,
- Values: [][]byte{convert.Int64ToBytes(50)},
- },
- {
- Key: "duration",
- Op: modelv1.PairQuery_BINARY_OP_LE,
- Values: [][]byte{convert.Int64ToBytes(5000)},
- },
- },
- },
- want: roaring.NewPostingListWithInitialData(13, 12, 11, 14),
- },
- }
- s := setUpModules(tester)
- setupData(tester, s)
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := s.Search(*common.NewMetadataByNameAndGroup("sw", "default"), 0, 0, math.MaxInt64, tt.args.indexObjectName, tt.args.conditions)
- if (err != nil) != tt.wantErr {
- t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if !got.Equal(tt.want) {
- t.Errorf("Search() got = %v, want %v", got.ToSlice(), tt.want.ToSlice())
- }
- })
- }
-}
-
-func setupData(tester *assert.Assertions, s *service) {
- fields := []*Field{
- {
- ChunkID: common.ChunkID(1),
- Name: "endpoint",
- Value: []byte("/product"),
- },
- {
- ChunkID: common.ChunkID(2),
- Name: "endpoint",
- Value: []byte("/home"),
- },
- {
- ChunkID: common.ChunkID(3),
- Name: "endpoint",
- Value: []byte("/sales"),
- },
- {
- ChunkID: common.ChunkID(11),
- Name: "duration",
- Value: convert.Int64ToBytes(50),
- },
- {
- ChunkID: common.ChunkID(12),
- Name: "duration",
- Value: convert.Int64ToBytes(500),
- },
- {
- ChunkID: common.ChunkID(13),
- Name: "duration",
- Value: convert.Int64ToBytes(100),
- },
- {
- ChunkID: common.ChunkID(14),
- Name: "duration",
- Value: convert.Int64ToBytes(5000),
- },
- }
- for _, field := range fields {
- if err := s.Insert(*common.NewMetadataByNameAndGroup("sw", "default"), 0, field); err != nil {
- tester.NoError(err)
- }
- }
-}
diff --git a/banyand/index/tsdb/field_map.go b/banyand/index/tsdb/field_map.go
deleted file mode 100644
index 32c9488..0000000
--- a/banyand/index/tsdb/field_map.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
-)
-
-var ErrFieldAbsent = errors.New("field doesn't exist")
-
-type fieldHashID uint64
-
-type fieldMap struct {
- repo map[fieldHashID]*fieldValue
-}
-
-func newFieldMap(initialSize int) *fieldMap {
- return &fieldMap{
- repo: make(map[fieldHashID]*fieldValue, initialSize),
- }
-}
-
-func (fm *fieldMap) createKey(key []byte) {
- fm.repo[fieldHashID(convert.Hash(key))] = &fieldValue{
- key: key,
- value: newPostingMap(),
- }
-}
-
-func (fm *fieldMap) get(key []byte) (*fieldValue, bool) {
- v, ok := fm.repo[fieldHashID(convert.Hash(key))]
- return v, ok
-}
-
-func (fm *fieldMap) put(fv *Field, id common.ChunkID) error {
- pm, ok := fm.get(fv.Name)
- if !ok {
- return errors.Wrapf(ErrFieldAbsent, "filed term:%s", fv.Name)
- }
- return pm.value.put(fv.Value, id)
-}
-
-type fieldValue struct {
- key []byte
- value *postingMap
-}
diff --git a/banyand/index/tsdb/mem.go b/banyand/index/tsdb/mem.go
deleted file mode 100644
index 2e2ea9b..0000000
--- a/banyand/index/tsdb/mem.go
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-var ErrFieldsAbsent = errors.New("fields are absent")
-
-type MemTable struct {
- terms *fieldMap
- name string
- group string
- shardID uint
-}
-
-func NewMemTable(name, group string, shardID uint) *MemTable {
- return &MemTable{
- name: name,
- group: group,
- shardID: shardID,
- }
-}
-
-type Field struct {
- Name []byte
- Value []byte
-}
-
-type FieldSpec struct {
- Name string
-}
-
-func (m *MemTable) Initialize(fields []FieldSpec) error {
- if len(fields) < 1 {
- return ErrFieldsAbsent
- }
- m.terms = newFieldMap(len(fields))
- for _, f := range fields {
- m.terms.createKey([]byte(f.Name))
- }
- return nil
-}
-
-func (m *MemTable) Insert(field *Field, chunkID common.ChunkID) error {
- return m.terms.put(field, chunkID)
-}
-
-func (m *MemTable) MatchField(fieldName []byte) (list posting.List) {
- fieldsValues, ok := m.terms.get(fieldName)
- if !ok {
- return roaring.EmptyPostingList
- }
- return fieldsValues.value.allValues()
-}
-
-func (m *MemTable) MatchTerms(field *Field) (list posting.List) {
- fieldsValues, ok := m.terms.get(field.Name)
- if !ok {
- return roaring.EmptyPostingList
- }
- return fieldsValues.value.get(field.Value).Clone()
-}
-
-func (m *MemTable) Range(fieldName []byte, opts *RangeOpts) (list posting.List) {
- fieldsValues, ok := m.terms.get(fieldName)
- if !ok {
- return roaring.EmptyPostingList
- }
- return fieldsValues.value.getRange(opts)
-}
diff --git a/banyand/index/tsdb/mem_test.go b/banyand/index/tsdb/mem_test.go
deleted file mode 100644
index fed7655..0000000
--- a/banyand/index/tsdb/mem_test.go
+++ /dev/null
@@ -1,261 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "reflect"
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-func TestMemTable_Initialize(t *testing.T) {
- type args struct {
- fields []FieldSpec
- }
- tests := []struct {
- name string
- args args
- wantErr bool
- }{
- {
- name: "golden path",
- args: args{
- fields: []FieldSpec{
- {
- Name: "service_name",
- },
- {
- Name: "duration",
- },
- },
- },
- },
- {
- name: "fields absent",
- wantErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- m := NewMemTable("sw", "group", 0)
- var err error
- if err = m.Initialize(tt.args.fields); (err != nil) != tt.wantErr {
- t.Errorf("Initialize() error = %v, wantErr %v", err, tt.wantErr)
- }
- if err != nil {
- return
- }
- assert.Equal(t, len(m.terms.repo), len(tt.args.fields))
- })
- }
-}
-
-func TestMemTable_Range(t *testing.T) {
- type args struct {
- fieldName []byte
- opts *RangeOpts
- }
- m := NewMemTable("sw", "group", 0)
- setUp(t, m)
- tests := []struct {
- name string
- args args
- wantList posting.List
- }{
- {
- name: "in range",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(100),
- Upper: convert.Uint16ToBytes(500),
- },
- },
- wantList: m.MatchTerms(&Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- }),
- },
- {
- name: "excludes edge",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(50),
- Upper: convert.Uint16ToBytes(1000),
- },
- },
- wantList: union(m,
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- },
- ),
- },
- {
- name: "includes lower",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(50),
- Upper: convert.Uint16ToBytes(1000),
- IncludesLower: true,
- },
- },
- wantList: union(m,
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(50),
- },
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- },
- ),
- },
- {
- name: "includes upper",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(50),
- Upper: convert.Uint16ToBytes(1000),
- IncludesUpper: true,
- },
- },
- wantList: union(m,
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- },
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(1000),
- },
- ),
- },
- {
- name: "includes edges",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(50),
- Upper: convert.Uint16ToBytes(1000),
- IncludesUpper: true,
- IncludesLower: true,
- },
- },
- wantList: union(m,
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(50),
- },
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- },
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(1000),
- },
- ),
- },
- {
- name: "match one",
- args: args{
- fieldName: []byte("duration"),
- opts: &RangeOpts{
- Lower: convert.Uint16ToBytes(200),
- Upper: convert.Uint16ToBytes(200),
- IncludesUpper: true,
- IncludesLower: true,
- },
- },
- wantList: union(m,
- &Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- },
- ),
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if gotList := m.Range(tt.args.fieldName, tt.args.opts); !reflect.DeepEqual(gotList, tt.wantList) {
- t.Errorf("Range() = %v, want %v", gotList.Len(), tt.wantList.Len())
- }
- })
- }
-}
-
-func union(memTable *MemTable, fields ...*Field) posting.List {
- result := roaring.NewPostingList()
- for _, f := range fields {
- _ = result.Union(memTable.MatchTerms(f))
- }
- return result
-}
-
-func setUp(t *testing.T, mt *MemTable) {
- assert.NoError(t, mt.Initialize([]FieldSpec{
- {
- Name: "service_name",
- },
- {
- Name: "duration",
- },
- }))
- for i := 0; i < 100; i++ {
- if i%2 == 0 {
- assert.NoError(t, mt.Insert(&Field{
- Name: []byte("service_name"),
- Value: []byte("gateway"),
- }, common.ChunkID(i)))
- } else {
- assert.NoError(t, mt.Insert(&Field{
- Name: []byte("service_name"),
- Value: []byte("webpage"),
- }, common.ChunkID(i)))
- }
- }
- for i := 100; i < 200; i++ {
- switch {
- case i%3 == 0:
- assert.NoError(t, mt.Insert(&Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(50),
- }, common.ChunkID(i)))
- case i%3 == 1:
- assert.NoError(t, mt.Insert(&Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(200),
- }, common.ChunkID(i)))
- case i%3 == 2:
- assert.NoError(t, mt.Insert(&Field{
- Name: []byte("duration"),
- Value: convert.Uint16ToBytes(1000),
- }, common.ChunkID(i)))
- }
- }
-}
diff --git a/banyand/index/tsdb/term_map.go b/banyand/index/tsdb/term_map.go
deleted file mode 100644
index 20bc492..0000000
--- a/banyand/index/tsdb/term_map.go
+++ /dev/null
@@ -1,143 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "bytes"
- "sort"
- "sync"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-type termHashID uint64
-
-type postingMap struct {
- repo map[termHashID]*postingValue
- mutex sync.RWMutex
-}
-
-func newPostingMap() *postingMap {
- return &postingMap{
- repo: make(map[termHashID]*postingValue),
- }
-}
-
-func (p *postingMap) put(key []byte, id common.ChunkID) error {
- list := p.getOrCreate(key)
- list.Insert(id)
- return nil
-}
-
-func (p *postingMap) getOrCreate(key []byte) posting.List {
- list := p.get(key)
- if list != roaring.EmptyPostingList {
- return list
- }
- p.mutex.Lock()
- defer p.mutex.Unlock()
- hashedKey := termHashID(convert.Hash(key))
- v := &postingValue{
- key: key,
- value: roaring.NewPostingList(),
- }
- p.repo[hashedKey] = v
- return v.value
-}
-
-func (p *postingMap) get(key []byte) posting.List {
- p.mutex.RLock()
- defer p.mutex.RUnlock()
- hashedKey := termHashID(convert.Hash(key))
- v, ok := p.repo[hashedKey]
- if !ok {
- return roaring.EmptyPostingList
- }
- return v.value
-}
-
-func (p *postingMap) allValues() posting.List {
- result := roaring.NewPostingList()
- for _, value := range p.repo {
- _ = result.Union(value.value)
- }
- return result
-}
-
-func (p *postingMap) getRange(opts *RangeOpts) posting.List {
- switch bytes.Compare(opts.Upper, opts.Lower) {
- case -1:
- return roaring.EmptyPostingList
- case 0:
- if opts.IncludesUpper && opts.IncludesLower {
- return p.get(opts.Upper)
- }
- return roaring.EmptyPostingList
- }
- p.mutex.RLock()
- defer p.mutex.RUnlock()
- keys := make(Asc, 0, len(p.repo))
- for _, v := range p.repo {
- keys = append(keys, v.key)
- }
- sort.Sort(keys)
- index := sort.Search(len(keys), func(i int) bool {
- return bytes.Compare(keys[i], opts.Lower) >= 0
- })
- result := roaring.NewPostingList()
- for i := index; i < len(keys); i++ {
- k := keys[i]
- switch {
- case bytes.Equal(k, opts.Lower):
- if opts.IncludesLower {
- _ = result.Union(p.repo[termHashID(convert.Hash(k))].value)
- }
- case bytes.Compare(k, opts.Upper) > 0:
- break
- case bytes.Equal(k, opts.Upper):
- if opts.IncludesUpper {
- _ = result.Union(p.repo[termHashID(convert.Hash(k))].value)
- }
- default:
- _ = result.Union(p.repo[termHashID(convert.Hash(k))].value)
- }
- }
- return result
-}
-
-type Asc [][]byte
-
-func (a Asc) Len() int {
- return len(a)
-}
-
-func (a Asc) Less(i, j int) bool {
- return bytes.Compare(a[i], a[j]) < 0
-}
-
-func (a Asc) Swap(i, j int) {
- a[i], a[j] = a[j], a[i]
-}
-
-type postingValue struct {
- key []byte
- value posting.List
-}
diff --git a/banyand/index/tsdb/tsdb.go b/banyand/index/tsdb/tsdb.go
deleted file mode 100644
index 91f114d..0000000
--- a/banyand/index/tsdb/tsdb.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package tsdb
-
-import (
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/pkg/posting"
-)
-
-type RangeOpts struct {
- Upper []byte
- Lower []byte
- IncludesUpper bool
- IncludesLower bool
-}
-
-type GlobalStore interface {
- Window(startTime, endTime uint64) (Searcher, bool)
- Initialize(fields []FieldSpec) error
- Insert(field *Field, chunkID common.ChunkID) error
-}
-
-type Searcher interface {
- MatchField(fieldNames []byte) (list posting.List)
- MatchTerms(field *Field) (list posting.List)
- Range(fieldName []byte, opts *RangeOpts) (list posting.List)
-}
-
-type store struct {
- memTable *MemTable
- //TODO: add data tables
-}
-
-func (s *store) Window(_, _ uint64) (Searcher, bool) {
- return s.memTable, true
-}
-
-func (s *store) Initialize(fields []FieldSpec) error {
- return s.memTable.Initialize(fields)
-}
-
-func (s *store) Insert(field *Field, chunkID common.ChunkID) error {
- return s.memTable.Insert(field, chunkID)
-}
-
-func NewStore(name, group string, shardID uint) GlobalStore {
- return &store{
- memTable: NewMemTable(name, group, shardID),
- }
-}
diff --git a/banyand/internal/cmd/standalone.go b/banyand/internal/cmd/standalone.go
index 254dabc..19c3cf1 100644
--- a/banyand/internal/cmd/standalone.go
+++ b/banyand/internal/cmd/standalone.go
@@ -56,7 +56,7 @@ func newStandaloneCmd() *cobra.Command {
if err != nil {
l.Fatal().Err(err).Msg("failed to initiate metadata service")
}
- streamSvc, err := stream.NewService(ctx, metaSvc, pipeline)
+ streamSvc, err := stream.NewService(ctx, metaSvc, repo, pipeline)
if err != nil {
l.Fatal().Err(err).Msg("failed to initiate metadata service")
}
@@ -73,6 +73,7 @@ func newStandaloneCmd() *cobra.Command {
g.Register(
new(signal.Handler),
repo,
+ pipeline,
metaSvc,
streamSvc,
q,
diff --git a/banyand/liaison/grpc/discovery.go b/banyand/liaison/grpc/discovery.go
new file mode 100644
index 0000000..c0f222c
--- /dev/null
+++ b/banyand/liaison/grpc/discovery.go
@@ -0,0 +1,126 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "sync"
+
+ commonv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
+ databasev2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v2"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/partition"
+)
+
+type identity struct {
+ name string
+ group string
+}
+
+type shardRepo struct {
+ log *logger.Logger
+ shardEventsMap map[identity]uint32
+ sync.RWMutex
+}
+
+func (s *shardRepo) Rev(message bus.Message) (resp bus.Message) {
+ e, ok := message.Data().(*databasev2.ShardEvent)
+ if !ok {
+ s.log.Warn().Msg("invalid e data type")
+ return
+ }
+ s.setShardNum(e)
+ s.log.Info().
+ Str("action", databasev2.Action_name[int32(e.Action)]).
+ Uint64("shardID", e.Shard.Id).
+ Msg("received a shard e")
+ return
+}
+
+func (s *shardRepo) setShardNum(eventVal *databasev2.ShardEvent) {
+ s.RWMutex.Lock()
+ defer s.RWMutex.Unlock()
+ idx := getID(eventVal.GetShard().GetMetadata())
+ if eventVal.Action == databasev2.Action_ACTION_PUT {
+ s.shardEventsMap[idx] = eventVal.Shard.Total
+ } else if eventVal.Action == databasev2.Action_ACTION_DELETE {
+ delete(s.shardEventsMap, idx)
+ }
+}
+
+func (s *shardRepo) shardNum(idx identity) (uint32, bool) {
+ s.RWMutex.RLock()
+ defer s.RWMutex.RUnlock()
+ sn, ok := s.shardEventsMap[idx]
+ if !ok {
+ return 0, false
+ }
+ return sn, true
+}
+
+func getID(metadata *commonv2.Metadata) identity {
+ return identity{
+ name: metadata.GetName(),
+ group: metadata.GetGroup(),
+ }
+}
+
+type entityRepo struct {
+ log *logger.Logger
+ entitiesMap map[identity]partition.EntityLocator
+ sync.RWMutex
+}
+
+func (s *entityRepo) Rev(message bus.Message) (resp bus.Message) {
+ e, ok := message.Data().(*databasev2.EntityEvent)
+ if !ok {
+ s.log.Warn().Msg("invalid e data type")
+ return
+ }
+ id := getID(e.GetSubject())
+ s.log.Info().
+ Str("action", databasev2.Action_name[int32(e.Action)]).
+ Interface("subject", id).
+ Msg("received an entity event")
+ s.RWMutex.Lock()
+ defer s.RWMutex.Unlock()
+ switch e.Action {
+ case databasev2.Action_ACTION_PUT:
+ en := make(partition.EntityLocator, 0, len(e.GetEntityLocator()))
+ for _, l := range e.GetEntityLocator() {
+ en = append(en, partition.TagLocator{
+ FamilyOffset: int(l.FamilyOffset),
+ TagOffset: int(l.TagOffset),
+ })
+ }
+ s.entitiesMap[id] = en
+ case databasev2.Action_ACTION_DELETE:
+ delete(s.entitiesMap, id)
+ }
+ return
+}
+
+func (s *entityRepo) getLocator(id identity) (partition.EntityLocator, bool) {
+ s.RWMutex.RLock()
+ defer s.RWMutex.RUnlock()
+ el, ok := s.entitiesMap[id]
+ if !ok {
+ return nil, false
+ }
+ return el, true
+}
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
new file mode 100644
index 0000000..a4617dd
--- /dev/null
+++ b/banyand/liaison/grpc/server.go
@@ -0,0 +1,139 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "context"
+ "net"
+
+ "github.com/pkg/errors"
+ grpclib "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+
+ "github.com/apache/skywalking-banyandb/api/event"
+ streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
+ "github.com/apache/skywalking-banyandb/banyand/discovery"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/partition"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+const defaultRecvSize = 1024 * 1024 * 10
+
+var (
+ ErrServerCert = errors.New("invalid server cert file")
+ ErrServerKey = errors.New("invalid server key file")
+ ErrNoAddr = errors.New("no address")
+ ErrQueryMsg = errors.New("invalid query message")
+)
+
+type Server struct {
+ addr string
+ maxRecvMsgSize int
+ tls bool
+ certFile string
+ keyFile string
+ log *logger.Logger
+ ser *grpclib.Server
+ pipeline queue.Queue
+ repo discovery.ServiceRepo
+ streamv2.UnimplementedStreamServiceServer
+ creds credentials.TransportCredentials
+ shardRepo *shardRepo
+ entityRepo *entityRepo
+}
+
+func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRepo) *Server {
+ return &Server{
+ pipeline: pipeline,
+ repo: repo,
+ shardRepo: &shardRepo{shardEventsMap: make(map[identity]uint32)},
+ entityRepo: &entityRepo{entitiesMap: make(map[identity]partition.EntityLocator)},
+ }
+}
+
+func (s *Server) PreRun() error {
+ s.log = logger.GetLogger("liaison-grpc")
+ s.shardRepo.log = s.log
+ s.entityRepo.log = s.log
+ err := s.repo.Subscribe(event.TopicShardEvent, s.shardRepo)
+ if err != nil {
+ return err
+ }
+ return s.repo.Subscribe(event.TopicEntityEvent, s.entityRepo)
+}
+
+func (s *Server) Name() string {
+ return "grpc"
+}
+
+func (s *Server) FlagSet() *run.FlagSet {
+ fs := run.NewFlagSet("grpc")
+ fs.IntVarP(&s.maxRecvMsgSize, "max-recv-msg-size", "", defaultRecvSize, "The size of max receiving message")
+ fs.BoolVarP(&s.tls, "tls", "", false, "Connection uses TLS if true, else plain TCP")
+ fs.StringVarP(&s.certFile, "cert-file", "", "", "The TLS cert file")
+ fs.StringVarP(&s.keyFile, "key-file", "", "", "The TLS key file")
+ fs.StringVarP(&s.addr, "addr", "", ":17912", "The address of banyand listens")
+ return fs
+}
+
+func (s *Server) Validate() error {
+ if s.addr == "" {
+ return ErrNoAddr
+ }
+ if !s.tls {
+ return nil
+ }
+ if s.certFile == "" {
+ return ErrServerCert
+ }
+ if s.keyFile == "" {
+ return ErrServerKey
+ }
+ creds, errTLS := credentials.NewServerTLSFromFile(s.certFile, s.keyFile)
+ if errTLS != nil {
+ return errors.Wrap(errTLS, "failed to load cert and key")
+ }
+ s.creds = creds
+ return nil
+}
+
+func (s *Server) Serve() error {
+ lis, err := net.Listen("tcp", s.addr)
+ if err != nil {
+ s.log.Fatal().Err(err).Msg("Failed to listen")
+ }
+ if errValidate := s.Validate(); errValidate != nil {
+ s.log.Fatal().Err(errValidate).Msg("Failed to validate data")
+ }
+ var opts []grpclib.ServerOption
+ if s.tls {
+ opts = []grpclib.ServerOption{grpclib.Creds(s.creds)}
+ }
+ opts = append(opts, grpclib.MaxRecvMsgSize(s.maxRecvMsgSize))
+ s.ser = grpclib.NewServer(opts...)
+ streamv2.RegisterStreamServiceServer(s.ser, s)
+ s.log.Info().Str("addr", s.addr).Msg("Listening to")
+ return s.ser.Serve(lis)
+}
+
+func (s *Server) GracefulStop() {
+ s.log.Info().Msg("stopping")
+ s.ser.GracefulStop()
+}
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
new file mode 100644
index 0000000..0e9f733
--- /dev/null
+++ b/banyand/liaison/grpc/stream.go
@@ -0,0 +1,84 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "context"
+ "io"
+ "time"
+
+ "github.com/apache/skywalking-banyandb/api/data"
+ streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
+ "github.com/apache/skywalking-banyandb/banyand/tsdb"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+)
+
+func (s *Server) Write(stream streamv2.StreamService_WriteServer) error {
+ for {
+ writeEntity, err := stream.Recv()
+ if err == io.EOF {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ id := getID(writeEntity.GetMetadata())
+ shardNum, existed := s.shardRepo.shardNum(id)
+ if !existed {
+ continue
+ }
+ locator, existed := s.entityRepo.getLocator(id)
+ if !existed {
+ continue
+ }
+ entity, shardID, err := locator.Locate(writeEntity.GetElement().TagFamilies, shardNum)
+ if err != nil {
+ s.log.Error().Err(err).Msg("failed to locate write target")
+ continue
+ }
+ message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), &streamv2.InternalWriteRequest{
+ Request: writeEntity,
+ ShardId: uint32(shardID),
+ SeriesHash: tsdb.HashEntity(entity),
+ })
+ _, errWritePub := s.pipeline.Publish(data.TopicStreamWrite, message)
+ if errWritePub != nil {
+ return errWritePub
+ }
+ if errSend := stream.Send(&streamv2.WriteResponse{}); errSend != nil {
+ return errSend
+ }
+ }
+}
+
+func (s *Server) Query(_ context.Context, entityCriteria *streamv2.QueryRequest) (*streamv2.QueryResponse, error) {
+ message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), entityCriteria)
+ feat, errQuery := s.pipeline.Publish(data.TopicStreamQuery, message)
+ if errQuery != nil {
+ return nil, errQuery
+ }
+ msg, errFeat := feat.Get()
+ if errFeat != nil {
+ return nil, errFeat
+ }
+ queryMsg, ok := msg.Data().([]*streamv2.Element)
+ if !ok {
+ return nil, ErrQueryMsg
+ }
+ return &streamv2.QueryResponse{Elements: queryMsg}, nil
+}
diff --git a/banyand/liaison/grpc/stream_test.go b/banyand/liaison/grpc/stream_test.go
new file mode 100644
index 0000000..fcb26b3
--- /dev/null
+++ b/banyand/liaison/grpc/stream_test.go
@@ -0,0 +1,259 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "io"
+ "path/filepath"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ grpclib "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+
+ streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
+ "github.com/apache/skywalking-banyandb/banyand/discovery"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ query "github.com/apache/skywalking-banyandb/banyand/query/v2"
+ "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/stream"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+ v2 "github.com/apache/skywalking-banyandb/pkg/pb/v2"
+ "github.com/apache/skywalking-banyandb/pkg/run"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+type testData struct {
+ certFile string
+ serverHostOverride string
+ TLS bool
+ addr string
+ basePath string
+}
+
+func setup(req *require.Assertions, testData testData) func() {
+ req.NoError(logger.Init(logger.Logging{
+ Env: "dev",
+ Level: "warn",
+ }))
+ g := run.Group{Name: "standalone"}
+ // Init `Discovery` module
+ repo, err := discovery.NewServiceRepo(context.Background())
+ req.NoError(err)
+ // Init `Queue` module
+ pipeline, err := queue.NewQueue(context.TODO(), repo)
+ req.NoError(err)
+ // Init `Metadata` module
+ metaSvc, err := metadata.NewService(context.TODO())
+ req.NoError(err)
+ streamSvc, err := stream.NewService(context.TODO(), metaSvc, repo, pipeline)
+ req.NoError(err)
+ q, err := query.NewExecutor(context.TODO(), streamSvc, repo, pipeline)
+ req.NoError(err)
+
+ tcp := NewServer(context.TODO(), pipeline, repo)
+
+ closer := run.NewTester("closer")
+ startListener := run.NewTester("started-listener")
+ g.Register(
+ closer,
+ repo,
+ pipeline,
+ metaSvc,
+ streamSvc,
+ q,
+ tcp,
+ startListener,
+ )
+ // Create a random directory
+ rootPath, deferFunc := test.Space(req)
+ flags := []string{"--root-path=" + rootPath}
+ if testData.TLS {
+ flags = append(flags, "--tls=true")
+ certFile := filepath.Join(testData.basePath, "testdata/server_cert.pem")
+ keyFile := filepath.Join(testData.basePath, "testdata/server_key.pem")
+ flags = append(flags, "--cert-file="+certFile)
+ flags = append(flags, "--key-file="+keyFile)
+ flags = append(flags, "--addr="+testData.addr)
+ }
+ err = g.RegisterFlags().Parse(flags)
+ req.NoError(err)
+
+ go func() {
+ errRun := g.Run()
+ if errRun != nil {
+ startListener.GracefulStop()
+ req.NoError(errRun)
+ }
+ deferFunc()
+ }()
+ req.NoError(startListener.WaitUntilStarted())
+ return func() {
+ closer.GracefulStop()
+ }
+}
+
+type caseData struct {
+ name string
+ queryGenerator func(baseTs time.Time) *streamv2.QueryRequest
+ writeGenerator func() *streamv2.WriteRequest
+ args testData
+ wantLen int
+}
+
+func TestStreamService(t *testing.T) {
+ req := require.New(t)
+ _, currentFile, _, _ := runtime.Caller(0)
+ basePath := filepath.Dir(currentFile)
+ certFile := filepath.Join(basePath, "testdata/server_cert.pem")
+ testCases := []caseData{
+ {
+ name: "isTLS",
+ queryGenerator: queryCriteria,
+ writeGenerator: writeData,
+ args: testData{
+ TLS: true,
+ certFile: certFile,
+ serverHostOverride: "localhost",
+ addr: "localhost:17913",
+ basePath: basePath,
+ },
+ wantLen: 1,
+ },
+ {
+ name: "noTLS",
+ queryGenerator: queryCriteria,
+ writeGenerator: writeData,
+ args: testData{
+ TLS: false,
+ addr: "localhost:17912",
+ },
+ wantLen: 1,
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ gracefulStop := setup(req, tc.args)
+ defer gracefulStop()
+ if tc.args.TLS {
+ var opts []grpclib.DialOption
+ creds, err := credentials.NewClientTLSFromFile(tc.args.certFile, tc.args.serverHostOverride)
+ assert.NoError(t, err)
+ opts = append(opts, grpclib.WithTransportCredentials(creds))
+ dialService(t, tc, opts)
+ } else {
+ var opts []grpclib.DialOption
+ opts = append(opts, grpclib.WithInsecure())
+ dialService(t, tc, opts)
+ }
+ })
+ }
+}
+
+func writeData() *streamv2.WriteRequest {
+ bb, _ := base64.StdEncoding.DecodeString("YWJjMTIzIT8kKiYoKSctPUB+")
+ return v2.NewStreamWriteRequestBuilder().
+ ID("1").
+ Metadata("default", "sw").
+ Timestamp(time.Now()).
+ TagFamily(bb).
+ TagFamily(
+ "trace_id-xxfff.111",
+ 0,
+ "webapp_id",
+ "10.0.0.1_id",
+ "/home_id",
+ 300,
+ 1622933202000000000,
+ ).
+ Build()
+}
+
+func queryCriteria(baseTs time.Time) *streamv2.QueryRequest {
+ return v2.NewQueryRequestBuilder().
+ Limit(10).
+ Offset(0).
+ Metadata("default", "sw").
+ TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
+ Projection("searchable", "trace_id").
+ Build()
+}
+
+func dialService(t *testing.T, tc caseData, opts []grpclib.DialOption) {
+ conn, err := grpclib.Dial(tc.args.addr, opts...)
+ assert.NoError(t, err)
+ defer func(conn *grpclib.ClientConn) {
+ _ = conn.Close()
+ }(conn)
+ streamWrite(t, tc, conn)
+ requireTester := require.New(t)
+ assert.NoError(t, test.Retry(10, 100*time.Millisecond, func() error {
+ now := time.Now()
+ resp := streamQuery(requireTester, conn, tc.queryGenerator(now))
+ if len(resp.GetElements()) == tc.wantLen {
+ return nil
+ }
+ return fmt.Errorf("expected elements number: %d got: %d", tc.wantLen, len(resp.GetElements()))
+ }))
+}
+
+func streamWrite(t *testing.T, tc caseData, conn *grpclib.ClientConn) {
+ client := streamv2.NewStreamServiceClient(conn)
+ ctx := context.Background()
+ writeClient, errorWrite := client.Write(ctx)
+ if errorWrite != nil {
+ t.Errorf("%v.write(_) = _, %v", client, errorWrite)
+ }
+ waitc := make(chan struct{})
+ go func() {
+ for {
+ writeResponse, errRecv := writeClient.Recv()
+ if errRecv == io.EOF {
+ // read done.
+ close(waitc)
+ return
+ }
+ assert.NoError(t, errRecv)
+ assert.NotNil(t, writeResponse)
+ }
+ }()
+ if errSend := writeClient.Send(tc.writeGenerator()); errSend != nil {
+ t.Errorf("Failed to send a note: %v", errSend)
+ }
+ if errorSend := writeClient.CloseSend(); errorSend != nil {
+ t.Errorf("Failed to send a note: %v", errorSend)
+ }
+ <-waitc
+}
+
+func streamQuery(tester *require.Assertions, conn *grpclib.ClientConn, request *streamv2.QueryRequest) *streamv2.QueryResponse {
+ client := streamv2.NewStreamServiceClient(conn)
+ ctx := context.Background()
+ queryResponse, errRev := client.Query(ctx, request)
+ if errRev != nil {
+ tester.Errorf(errRev, "Retrieve client failed: %v")
+ }
+ tester.NotNil(queryResponse)
+ return queryResponse
+}
diff --git a/banyand/liaison/grpc/trace.go b/banyand/liaison/grpc/trace.go
deleted file mode 100644
index dd55aad..0000000
--- a/banyand/liaison/grpc/trace.go
+++ /dev/null
@@ -1,347 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package grpc
-
-import (
- "context"
- "io"
- "net"
- "strconv"
- "sync"
- "time"
-
- "github.com/pkg/errors"
- grpclib "google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- "github.com/apache/skywalking-banyandb/api/event"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- apischema "github.com/apache/skywalking-banyandb/api/schema"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- "github.com/apache/skywalking-banyandb/pkg/query/v1/logical"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-var (
- ErrSeriesEvents = errors.New("no seriesEvent")
- ErrShardEvents = errors.New("no shardEvent")
- ErrInvalidSeriesID = errors.New("invalid seriesID")
- ErrServerCert = errors.New("invalid server cert file")
- ErrServerKey = errors.New("invalid server key file")
- ErrNoAddr = errors.New("no address")
- ErrQueryMsg = errors.New("invalid query message")
-
- defaultRecvSize = 1024 * 1024 * 10
-)
-
-type Server struct {
- addr string
- maxRecvMsgSize int
- tls bool
- certFile string
- keyFile string
- log *logger.Logger
- ser *grpclib.Server
- pipeline queue.Queue
- repo discovery.ServiceRepo
- shardInfo *shardInfo
- seriesInfo *seriesInfo
- tracev1.UnimplementedTraceServiceServer
- creds credentials.TransportCredentials
-}
-
-type shardInfo struct {
- log *logger.Logger
- shardEventsMap map[string]uint32
- sync.RWMutex
-}
-
-func (s *shardInfo) Rev(message bus.Message) (resp bus.Message) {
- e, ok := message.Data().(*databasev1.ShardEvent)
- if !ok {
- s.log.Warn().Msg("invalid e data type")
- return
- }
- s.setShardNum(e)
- s.log.Info().
- Str("action", databasev1.Action_name[int32(e.Action)]).
- Uint64("shardID", e.Shard.Id).
- Msg("received a shard e")
- return
-}
-
-func (s *shardInfo) setShardNum(eventVal *databasev1.ShardEvent) {
- s.RWMutex.Lock()
- defer s.RWMutex.Unlock()
- idx := eventVal.Shard.Series.GetName() + "-" + eventVal.Shard.Series.GetGroup()
- if eventVal.Action == databasev1.Action_ACTION_PUT {
- s.shardEventsMap[idx] = eventVal.Shard.Total
- } else if eventVal.Action == databasev1.Action_ACTION_DELETE {
- delete(s.shardEventsMap, idx)
- }
-}
-
-func (s *shardInfo) shardNum(idx string) uint32 {
- s.RWMutex.RLock()
- defer s.RWMutex.RUnlock()
- return s.shardEventsMap[idx]
-}
-
-type seriesInfo struct {
- log *logger.Logger
- seriesEventsMap map[string][]int
- sync.RWMutex
-}
-
-func (s *seriesInfo) Rev(message bus.Message) (resp bus.Message) {
- e, ok := message.Data().(*databasev1.SeriesEvent)
- if !ok {
- s.log.Warn().Msg("invalid e data type")
- return
- }
- s.updateFieldIndexCompositeSeriesID(e)
- s.log.Info().
- Str("action", databasev1.Action_name[int32(e.Action)]).
- Str("name", e.Series.Name).
- Str("group", e.Series.Group).
- Msg("received a shard e")
- return
-}
-
-func (s *seriesInfo) updateFieldIndexCompositeSeriesID(seriesEventVal *databasev1.SeriesEvent) {
- s.RWMutex.Lock()
- defer s.RWMutex.Unlock()
- str := seriesEventVal.Series.GetName() + "-" + seriesEventVal.Series.GetGroup()
- if seriesEventVal.Action == databasev1.Action_ACTION_PUT {
- ana := logical.DefaultAnalyzer()
- metadata := common.Metadata{
- KindVersion: apischema.SeriesKindVersion,
- Spec: seriesEventVal.Series,
- }
- schema, err := ana.BuildTraceSchema(context.TODO(), metadata)
- if err != nil {
- s.log.Err(err).Msg("build trace schema")
- return
- }
- fieldRefs, errField := schema.CreateRef(seriesEventVal.FieldNamesCompositeSeriesId...)
- if errField != nil {
- s.log.Err(errField).Msg("create series ref")
- return
- }
- refIdx := make([]int, len(fieldRefs))
- for i, ref := range fieldRefs {
- refIdx[i] = ref.Spec.Idx
- }
- s.seriesEventsMap[str] = refIdx
- } else if seriesEventVal.Action == databasev1.Action_ACTION_DELETE {
- delete(s.seriesEventsMap, str)
- }
-}
-
-func (s *seriesInfo) FieldIndexCompositeSeriesID(seriesMeta string) []int {
- s.RWMutex.RLock()
- defer s.RWMutex.RUnlock()
- return s.seriesEventsMap[seriesMeta]
-}
-
-func (s *Server) PreRun() error {
- s.log = logger.GetLogger("liaison-grpc")
- s.shardInfo.log = s.log
- s.seriesInfo.log = s.log
- err := s.repo.Subscribe(event.TopicShardEvent, s.shardInfo)
- if err != nil {
- return err
- }
- return s.repo.Subscribe(event.TopicSeriesEvent, s.seriesInfo)
-}
-
-func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRepo) *Server {
- return &Server{
- pipeline: pipeline,
- repo: repo,
- shardInfo: &shardInfo{shardEventsMap: make(map[string]uint32)},
- seriesInfo: &seriesInfo{seriesEventsMap: make(map[string][]int)},
- }
-}
-
-func (s *Server) Name() string {
- return "grpc"
-}
-
-func (s *Server) FlagSet() *run.FlagSet {
- fs := run.NewFlagSet("grpc")
- fs.IntVarP(&s.maxRecvMsgSize, "max-recv-msg-size", "", defaultRecvSize, "The size of max receiving message")
- fs.BoolVarP(&s.tls, "tls", "", true, "Connection uses TLS if true, else plain TCP")
- fs.StringVarP(&s.certFile, "cert-file", "", "server_cert.pem", "The TLS cert file")
- fs.StringVarP(&s.keyFile, "key-file", "", "server_key.pem", "The TLS key file")
- fs.StringVarP(&s.addr, "addr", "", ":17912", "The address of banyand listens")
- return fs
-}
-
-func (s *Server) Validate() error {
- if s.addr == "" {
- return ErrNoAddr
- }
- if !s.tls {
- return nil
- }
- if s.certFile == "" {
- return ErrServerCert
- }
- if s.keyFile == "" {
- return ErrServerKey
- }
- creds, errTLS := credentials.NewServerTLSFromFile(s.certFile, s.keyFile)
- if errTLS != nil {
- return errTLS
- }
- s.creds = creds
- return nil
-}
-
-func (s *Server) Serve() error {
- lis, err := net.Listen("tcp", s.addr)
- if err != nil {
- s.log.Fatal().Err(err).Msg("Failed to listen")
- }
- if errValidate := s.Validate(); errValidate != nil {
- s.log.Fatal().Err(errValidate).Msg("Failed to validate data")
- }
- var opts []grpclib.ServerOption
- if s.tls {
- opts = []grpclib.ServerOption{grpclib.Creds(s.creds)}
- }
- opts = append(opts, grpclib.MaxRecvMsgSize(s.maxRecvMsgSize))
- s.ser = grpclib.NewServer(opts...)
- tracev1.RegisterTraceServiceServer(s.ser, s)
-
- return s.ser.Serve(lis)
-}
-
-func (s *Server) GracefulStop() {
- s.log.Info().Msg("stopping")
- s.ser.GracefulStop()
-}
-
-func (s *Server) computeSeriesID(writeEntity *tracev1.WriteRequest, mapIndexName string) ([]byte, error) {
- fieldNames := s.seriesInfo.FieldIndexCompositeSeriesID(mapIndexName)
- if fieldNames == nil {
- return nil, ErrSeriesEvents
- }
- var str string
- for _, ref := range fieldNames {
- field := writeEntity.GetEntity().GetFields()[ref]
- switch v := field.GetValueType().(type) {
- case *modelv1.Field_StrArray:
- for j := 0; j < len(v.StrArray.Value); j++ {
- str = str + v.StrArray.Value[j]
- }
- case *modelv1.Field_IntArray:
- for t := 0; t < len(v.IntArray.Value); t++ {
- str = str + strconv.FormatInt(v.IntArray.Value[t], 10)
- }
- case *modelv1.Field_Int:
- str = str + strconv.FormatInt(v.Int.Value, 10)
- case *modelv1.Field_Str:
- str = str + v.Str.Value
- }
- str = str + ":"
- }
- if str == "" {
- return nil, ErrInvalidSeriesID
- }
-
- return []byte(str), nil
-}
-
-func (s *Server) computeShardID(seriesID []byte, mapIndexName string) (uint, error) {
- shardNum := s.shardInfo.shardNum(mapIndexName)
- if shardNum < 1 {
- return 0, ErrShardEvents
- }
- shardID, shardIDError := partition.ShardID(seriesID, shardNum)
- if shardIDError != nil {
- return 0, shardIDError
- }
- return shardID, nil
-}
-
-func (s *Server) Write(stream tracev1.TraceService_WriteServer) error {
- for {
- writeEntity, err := stream.Recv()
- if err == io.EOF {
- return nil
- }
- if err != nil {
- return err
- }
- mapIndexName := writeEntity.GetMetadata().GetName() + "-" + writeEntity.GetMetadata().GetGroup()
- seriesID, err := s.computeSeriesID(writeEntity, mapIndexName)
- if err != nil {
- return err
- }
- shardID, err := s.computeShardID(seriesID, mapIndexName)
- if err != nil {
- return err
- }
- mergeData := assemblyWriteData(shardID, writeEntity, convert.BytesToUint64(seriesID))
- message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), mergeData)
- _, errWritePub := s.pipeline.Publish(data.TopicWriteEvent, message)
- if errWritePub != nil {
- return errWritePub
- }
- if errSend := stream.Send(&tracev1.WriteResponse{}); errSend != nil {
- return errSend
- }
- }
-}
-
-func (s *Server) Query(_ context.Context, entityCriteria *tracev1.QueryRequest) (*tracev1.QueryResponse, error) {
- message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), entityCriteria)
- feat, errQuery := s.pipeline.Publish(data.TopicQueryEvent, message)
- if errQuery != nil {
- return nil, errQuery
- }
- msg, errFeat := feat.Get()
- if errFeat != nil {
- return nil, errFeat
- }
- queryMsg, ok := msg.Data().([]data.Entity)
- if !ok {
- return nil, ErrQueryMsg
- }
- var arr []*tracev1.Entity
- for i := 0; i < len(queryMsg); i++ {
- arr = append(arr, queryMsg[i].Entity)
- }
-
- return &tracev1.QueryResponse{Entities: arr}, nil
-}
-
-func assemblyWriteData(shardID uint, writeEntity *tracev1.WriteRequest, seriesID uint64) data.TraceWriteDate {
- return data.TraceWriteDate{ShardID: shardID, SeriesID: seriesID, WriteRequest: writeEntity}
-}
diff --git a/banyand/liaison/grpc/trace_test.go b/banyand/liaison/grpc/trace_test.go
deleted file mode 100644
index 3c762d1..0000000
--- a/banyand/liaison/grpc/trace_test.go
+++ /dev/null
@@ -1,299 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package grpc_test
-
-import (
- "context"
- "io"
- "os"
- "path"
- "path/filepath"
- "runtime"
- "testing"
- "time"
-
- googleUUID "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- grpclib "google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
-
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
- v12 "github.com/apache/skywalking-banyandb/banyand/query/v1"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/series/trace"
- "github.com/apache/skywalking-banyandb/banyand/storage"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
-
-type testData struct {
- certFile string
- serverHostOverride string
- TLS bool
- addr string
-}
-
-func setup(tester *require.Assertions) (*grpc.Server, *grpc.Server, func()) {
- tester.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: "warn",
- }))
- // Init `Discovery` module
- repo, err := discovery.NewServiceRepo(context.Background())
- tester.NoError(err)
- tester.NotNil(repo)
- // Init `Queue` module
- pipeline, err := queue.NewQueue(context.TODO(), repo)
- tester.NoError(err)
- // Init `Database` module
- db, err := storage.NewDB(context.TODO(), repo)
- tester.NoError(err)
- uuid, err := googleUUID.NewUUID()
- tester.NoError(err)
- rootPath := path.Join(os.TempDir(), "banyandb-"+uuid.String())
- tester.NoError(db.FlagSet().Parse([]string{"--root-path=" + rootPath}))
- // Init `Index` module
- indexSvc, err := index.NewService(context.TODO(), repo)
- tester.NoError(err)
- // Init `Trace` module
- traceSvc, err := trace.NewService(context.TODO(), db, repo, indexSvc, pipeline)
- tester.NoError(err)
- // Init `Query` module
- executor, err := v12.NewExecutor(context.TODO(), repo, indexSvc, traceSvc, traceSvc, pipeline)
- tester.NoError(err)
- // Init `liaison` module
- tcp := grpc.NewServer(context.TODO(), pipeline, repo)
- tester.NoError(tcp.FlagSet().Parse([]string{"--tls=false", "--addr=:17912"}))
- tcpTLS := grpc.NewServer(context.TODO(), pipeline, repo)
- tester.NoError(tcpTLS.FlagSet().Parse([]string{"--tls=true", "--addr=:17913", "--cert-file=testdata/server_cert.pem", "--key-file=testdata/server_key.pem"}))
-
- err = indexSvc.PreRun()
- tester.NoError(err)
-
- err = traceSvc.PreRun()
- tester.NoError(err)
-
- err = db.PreRun()
- tester.NoError(err)
-
- err = executor.PreRun()
- tester.NoError(err)
-
- err = tcp.PreRun()
- tester.NoError(err)
- tester.NoError(tcpTLS.PreRun())
-
- go func() {
- tester.NoError(traceSvc.Serve())
- }()
-
- go func() {
- tester.NoError(tcpTLS.Serve())
- }()
-
- go func() {
- tester.NoError(tcp.Serve())
- }()
-
- ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancelFunc()
-
- tester.True(indexSvc.Ready(ctx, index.MetaExists("default", "sw")))
-
- return tcp, tcpTLS, func() {
- db.GracefulStop()
- _ = os.RemoveAll(rootPath)
- }
-}
-
-type caseData struct {
- name string
- queryGenerator func(baseTs time.Time) *tracev1.QueryRequest
- writeGenerator func() *tracev1.WriteRequest
- args testData
- wantLen int
-}
-
-func TestTraceService(t *testing.T) {
- tester := require.New(t)
- tcp, tcpTLS, gracefulStop := setup(tester)
- defer gracefulStop()
- _, currentFile, _, _ := runtime.Caller(0)
- basePath := filepath.Dir(currentFile)
- certFile := filepath.Join(basePath, "testdata/server_cert.pem")
- testCases := []caseData{
- {
- name: "isTLS",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("trace_id", "=", "trace_id-xxfff.111").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- writeGenerator: func() *tracev1.WriteRequest {
- entityValue := v1.NewEntityValueBuilder().
- EntityID("entityId123").
- DataBinary([]byte{12}).
- Fields("trace_id-xxfff.111",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000).
- Timestamp(time.Now()).
- Build()
- criteria := v1.NewWriteEntityBuilder().
- EntityValue(entityValue).
- Metadata("default", "sw").
- Build()
- return criteria
- },
- args: testData{
- TLS: true,
- certFile: certFile,
- serverHostOverride: "localhost",
- addr: "localhost:17913",
- },
- wantLen: 1,
- },
- {
- name: "noTLS",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("trace_id", "=", "trace_id-xxfff.111323").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- writeGenerator: func() *tracev1.WriteRequest {
- entityValue := v1.NewEntityValueBuilder().
- EntityID("entityId123").
- DataBinary([]byte{12}).
- Fields("trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000).
- Timestamp(time.Now()).
- Build()
- criteria := v1.NewWriteEntityBuilder().
- EntityValue(entityValue).
- Metadata("default", "sw").
- Build()
- return criteria
- },
- args: testData{
- TLS: false,
- addr: "localhost:17912",
- },
- wantLen: 1,
- },
- }
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- if tc.args.TLS {
- errValidate := tcpTLS.Validate()
- assert.NoError(t, errValidate)
- var opts []grpclib.DialOption
- creds, err := credentials.NewClientTLSFromFile(tc.args.certFile, tc.args.serverHostOverride)
- assert.NoError(t, err)
- opts = append(opts, grpclib.WithTransportCredentials(creds))
- dialService(t, tc, opts)
- } else {
- errValidate := tcp.Validate()
- assert.NoError(t, errValidate)
- var opts []grpclib.DialOption
- opts = append(opts, grpclib.WithInsecure())
- dialService(t, tc, opts)
- }
- })
- }
-}
-
-func dialService(t *testing.T, tc caseData, opts []grpclib.DialOption) {
- conn, err := grpclib.Dial(tc.args.addr, opts...)
- assert.NoError(t, err)
- defer conn.Close()
- traceWrite(t, tc, conn)
- requireTester := require.New(t)
- retry := 10
- for retry > 0 {
- now := time.Now()
- resp := traceQuery(requireTester, conn, tc.queryGenerator(now))
- if len(resp.GetEntities()) == tc.wantLen {
- break
- } else {
- time.Sleep(1 * time.Second)
- retry--
- }
- }
-}
-
-func traceWrite(t *testing.T, tc caseData, conn *grpclib.ClientConn) {
- client := tracev1.NewTraceServiceClient(conn)
- ctx := context.Background()
- stream, errorWrite := client.Write(ctx)
- if errorWrite != nil {
- t.Errorf("%v.write(_) = _, %v", client, errorWrite)
- }
- waitc := make(chan struct{})
- go func() {
- for {
- writeResponse, errRecv := stream.Recv()
- if errRecv == io.EOF {
- // read done.
- close(waitc)
- return
- }
- assert.NoError(t, errRecv)
- assert.NotNil(t, writeResponse)
- }
- }()
- if errSend := stream.Send(tc.writeGenerator()); errSend != nil {
- t.Errorf("Failed to send a note: %v", errSend)
- }
- if errorSend := stream.CloseSend(); errorSend != nil {
- t.Errorf("Failed to send a note: %v", errorSend)
- }
- <-waitc
-}
-
-func traceQuery(tester *require.Assertions, conn *grpclib.ClientConn, request *tracev1.QueryRequest) *tracev1.QueryResponse {
- client := tracev1.NewTraceServiceClient(conn)
- ctx := context.Background()
- stream, errRev := client.Query(ctx, request)
- if errRev != nil {
- tester.Errorf(errRev, "Retrieve client failed: %v")
- }
- tester.NotNil(stream)
- return stream
-}
diff --git a/banyand/query/v1/processor.go b/banyand/query/v1/processor.go
deleted file mode 100644
index a93cd31..0000000
--- a/banyand/query/v1/processor.go
+++ /dev/null
@@ -1,102 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package v1
-
-import (
- "context"
- "time"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- apischema "github.com/apache/skywalking-banyandb/api/schema"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/query/v1/executor"
- "github.com/apache/skywalking-banyandb/pkg/query/v1/logical"
-)
-
-const (
- moduleName = "query-processor"
-)
-
-var (
- _ Executor = (*queryProcessor)(nil)
- _ bus.MessageListener = (*queryProcessor)(nil)
- _ executor.ExecutionContext = (*queryProcessor)(nil)
-)
-
-type queryProcessor struct {
- index.Repo
- series.UniModel
- logger *logger.Logger
- schemaRepo series.SchemaRepo
- log *logger.Logger
- serviceRepo discovery.ServiceRepo
- pipeline queue.Queue
-}
-
-func (q *queryProcessor) Rev(message bus.Message) (resp bus.Message) {
- queryCriteria, ok := message.Data().(*v1.QueryRequest)
- if !ok {
- q.log.Warn().Msg("invalid event data type")
- return
- }
- q.log.Info().
- Msg("received a query event")
- analyzer := logical.DefaultAnalyzer()
- metadata := &common.Metadata{
- KindVersion: apischema.SeriesKindVersion,
- Spec: queryCriteria.GetMetadata(),
- }
- s, err := analyzer.BuildTraceSchema(context.TODO(), *metadata)
- if err != nil {
- q.logger.Error().Err(err).Msg("fail to build trace schema")
- return
- }
-
- p, err := analyzer.Analyze(context.TODO(), queryCriteria, metadata, s)
- if err != nil {
- q.logger.Error().Err(err).Msg("fail to analyze the query request")
- return
- }
-
- entities, err := p.Execute(q)
- if err != nil {
- q.logger.Error().Err(err).Msg("fail to execute the query plan")
- return
- }
-
- now := time.Now().UnixNano()
- resp = bus.NewMessage(bus.MessageID(now), entities)
-
- return
-}
-
-func (q *queryProcessor) Name() string {
- return moduleName
-}
-
-func (q *queryProcessor) PreRun() error {
- q.log = logger.GetLogger(moduleName)
- return q.pipeline.Subscribe(data.TopicQueryEvent, q)
-}
diff --git a/banyand/query/v1/processor_test.go b/banyand/query/v1/processor_test.go
deleted file mode 100644
index 49feabb..0000000
--- a/banyand/query/v1/processor_test.go
+++ /dev/null
@@ -1,504 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package v1
-
-import (
- "context"
- "os"
- "path"
- "testing"
- "time"
-
- googleUUID "github.com/google/uuid"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/liaison/grpc"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/banyand/series/trace"
- "github.com/apache/skywalking-banyandb/banyand/storage"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/query/v1/logical"
-)
-
-var (
- interval = time.Millisecond * 500
- withoutDataBinaryChecker = func(entities []data.Entity) bool {
- for _, entity := range entities {
- if entity.DataBinary != nil {
- return false
- }
- }
- return true
- }
- withDataBinaryChecker = func(entities []data.Entity) bool {
- for _, entity := range entities {
- if entity.DataBinary == nil || len(entity.GetDataBinary()) == 0 {
- return false
- }
- }
- return true
- }
-)
-
-type entityValue struct {
- seriesID string
- entityID string
- dataBinary []byte
- ts time.Time
- items []interface{}
-}
-
-func setupServices(t *testing.T, tester *require.Assertions) (series.Service, queue.Queue, func()) {
- // Bootstrap logger system
- tester.NoError(logger.Init(logger.Logging{
- Env: "dev",
- Level: "warn",
- }))
-
- // Init `Discovery` module
- repo, err := discovery.NewServiceRepo(context.Background())
- tester.NoError(err)
- tester.NotNil(repo)
- // Init `Queue` module
- pipeline, err := queue.NewQueue(context.TODO(), repo)
- tester.NoError(err)
-
- // Init `Index` module
- indexSvc, err := index.NewService(context.TODO(), repo)
- tester.NoError(err)
-
- // Init `Database` module
- db, err := storage.NewDB(context.TODO(), repo)
- tester.NoError(err)
- uuid, err := googleUUID.NewUUID()
- tester.NoError(err)
- rootPath := path.Join(os.TempDir(), "banyandb-"+uuid.String())
- tester.NoError(db.FlagSet().Parse([]string{"--root-path=" + rootPath}))
-
- // Init `Trace` module
- traceSvc, err := trace.NewService(context.TODO(), db, repo, indexSvc, pipeline)
- tester.NoError(err)
-
- // Init `Query` module
- executor, err := NewExecutor(context.TODO(), repo, indexSvc, traceSvc, traceSvc, pipeline)
- tester.NoError(err)
-
- // Init `Liaison` module
- liaison := grpc.NewServer(context.TODO(), pipeline, repo)
-
- // :PreRun:
- // 1) TraceSeries,
- // 2) Database
- // 3) Index
- err = traceSvc.PreRun()
- tester.NoError(err)
-
- err = db.PreRun()
- tester.NoError(err)
-
- err = indexSvc.PreRun()
- tester.NoError(err)
-
- err = executor.PreRun()
- tester.NoError(err)
-
- err = liaison.PreRun()
- tester.NoError(err)
-
- // :Serve:
- go func() {
- err = traceSvc.Serve()
- tester.NoError(err)
- }()
-
- ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancelFunc()
- tester.True(indexSvc.Ready(ctx, index.MetaExists("default", "sw")))
-
- return traceSvc, pipeline, func() {
- db.GracefulStop()
- _ = os.RemoveAll(rootPath)
- }
-}
-
-func setupData(tester *require.Assertions, baseTs time.Time, svc series.Service) {
- metadata := common.Metadata{
- Spec: &commonv1.Metadata{
- Name: "sw",
- Group: "default",
- },
- }
-
- entityValues := []entityValue{
- {
- ts: baseTs,
- seriesID: "webapp_10.0.0.1",
- entityID: "1",
- dataBinary: []byte{11},
- items: []interface{}{
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 400,
- 1622933202000000000,
- },
- },
- {
- ts: baseTs.Add(interval),
- seriesID: "gateway_10.0.0.2",
- entityID: "2",
- dataBinary: []byte{12},
- items: []interface{}{
- "trace_id-xxfff.111323a",
- 1,
- },
- },
- {
- ts: baseTs.Add(interval * 2),
- seriesID: "httpserver_10.0.0.3",
- entityID: "3",
- dataBinary: []byte{13},
- items: []interface{}{
- "trace_id-xxfff.111323",
- 1,
- "httpserver_id",
- "10.0.0.3_id",
- "/home_id",
- 300,
- 1622933202000000000,
- "GET",
- "200",
- },
- },
- {
- ts: baseTs.Add(interval * 3),
- seriesID: "database_10.0.0.4",
- entityID: "4",
- dataBinary: []byte{14},
- items: []interface{}{
- "trace_id-xxfff.111323",
- 0,
- "database_id",
- "10.0.0.4_id",
- "/home_id",
- 350,
- 1622933202000000000,
- nil,
- nil,
- "MySQL",
- "10.1.1.4",
- },
- },
- {
- ts: baseTs.Add(interval * 4),
- seriesID: "mq_10.0.0.5",
- entityID: "5",
- dataBinary: []byte{15},
- items: []interface{}{
- "trace_id-zzpp.111323",
- 0,
- "mq_id",
- "10.0.0.5_id",
- "/home_id",
- 302,
- 1622933202000000000,
- nil,
- nil,
- nil,
- nil,
- "test_topic",
- "10.0.0.5",
- },
- },
- {
- ts: baseTs.Add(interval * 5),
- seriesID: "database_10.0.0.6",
- entityID: "6",
- dataBinary: []byte{16},
- items: []interface{}{
- "trace_id-zzpp.111323",
- 1,
- "database_id",
- "10.0.0.6_id",
- "/home_id",
- 200,
- 1622933202000000000,
- nil,
- nil,
- "MySQL",
- "10.1.1.6",
- },
- },
- {
- ts: baseTs.Add(interval * 6),
- seriesID: "mq_10.0.0.7",
- entityID: "7",
- dataBinary: []byte{17},
- items: []interface{}{
- "trace_id-zzpp.111323",
- 0,
- "nq_id",
- "10.0.0.7_id",
- "/home_id",
- 100,
- 1622933202000000000,
- nil,
- nil,
- nil,
- nil,
- "test_topic",
- "10.0.0.7",
- },
- },
- }
-
- for _, ev := range entityValues {
- ok, err := svc.Write(metadata, ev.ts, ev.seriesID, ev.entityID, ev.dataBinary, ev.items...)
- tester.True(ok)
- tester.NoError(err)
- }
-}
-
-func TestQueryProcessor(t *testing.T) {
- tester := require.New(t)
-
- // setup services
- traceSvc, pipeline, gracefulStop := setupServices(t, tester)
- defer gracefulStop()
-
- baseTs := time.Now()
- setupData(tester, baseTs, traceSvc)
-
- tests := []struct {
- // name of the test case
- name string
- // queryGenerator is used to generate a Query
- queryGenerator func(baseTs time.Time) *tracev1.QueryRequest
- // wantLen is the length of entities expected to return
- wantLen int
- // checker is the customized checker for extra checks
- checker func([]data.Entity) bool
- }{
- {
- name: "query given timeRange is out of the time range of data",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- TimeRange(time.Unix(0, 0), time.Unix(0, 1)).
- Projection("trace_id").
- Build()
- },
- wantLen: 0,
- },
- {
- name: "query given timeRange which slightly covers the first three segments",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- TimeRange(baseTs.Add(-1*time.Nanosecond), baseTs.Add(2*interval).Add(1*time.Nanosecond)).
- Projection("trace_id").
- Build()
- },
- wantLen: 3,
- checker: withoutDataBinaryChecker,
- },
- {
- name: "query given timeRange which slightly covers the first three segments with data binary projection",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- TimeRange(baseTs.Add(-1*time.Nanosecond), baseTs.Add(2*interval).Add(1*time.Nanosecond)).
- ProjectionWithDataBinary("trace_id").
- Build()
- },
- wantLen: 3,
- checker: withDataBinaryChecker,
- },
- {
- name: "query given timeRange which slightly covers the first three segments ans sort by duration",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- TimeRange(baseTs.Add(-1*time.Nanosecond), baseTs.Add(2*interval).Add(1*time.Nanosecond)).
- OrderBy("duration", modelv1.QueryOrder_SORT_DESC).
- Projection("trace_id", "duration").
- Build()
- },
- wantLen: 3,
- checker: func(entities []data.Entity) bool {
- return logical.Sorted(entities, 1, modelv1.QueryOrder_SORT_DESC)
- },
- },
- {
- name: "query TraceID given timeRange includes the time range of data",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("trace_id", "=", "trace_id-zzpp.111323").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 3,
- checker: withoutDataBinaryChecker,
- },
- {
- name: "query TraceID given timeRange includes the time range of data with dataBinary projection",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("trace_id", "=", "trace_id-zzpp.111323").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- ProjectionWithDataBinary("trace_id").
- Build()
- },
- wantLen: 3,
- checker: withDataBinaryChecker,
- },
- {
- name: "query TraceID given timeRange includes the time range of data but limit to 1",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(1).
- Offset(0).
- Metadata("default", "sw").
- Fields("trace_id", "=", "trace_id-zzpp.111323").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 1,
- },
- {
- name: "Numerical Index - query duration < 100",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(1).
- Offset(0).
- Metadata("default", "sw").
- Fields("duration", "<", 100).
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 0,
- },
- {
- name: "Numerical Index - query duration <= 400",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("duration", "<=", 400).
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 6,
- },
- {
- name: "Textual Index - db.type == MySQL",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("db.type", "=", "MySQL").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 2,
- checker: withoutDataBinaryChecker,
- },
- {
- name: "Textual Index - db.type == MySQL with dataBinary projection",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("db.type", "=", "MySQL").
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- ProjectionWithDataBinary("trace_id").
- Build()
- },
- wantLen: 2,
- checker: withDataBinaryChecker,
- },
- {
- name: "Mixed Index - db.type == MySQL AND duration <= 300",
- queryGenerator: func(baseTs time.Time) *tracev1.QueryRequest {
- return v1.NewQueryRequestBuilder().
- Limit(10).
- Offset(0).
- Metadata("default", "sw").
- Fields("db.type", "=", "MySQL", "duration", "<=", 300).
- TimeRange(baseTs.Add(-1*time.Minute), baseTs.Add(1*time.Minute)).
- Projection("trace_id").
- Build()
- },
- wantLen: 1,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- singleTester := require.New(t)
- now := time.Now()
- m := bus.NewMessage(bus.MessageID(now.UnixNano()), tt.queryGenerator(baseTs))
- f, err := pipeline.Publish(data.TopicQueryEvent, m)
- singleTester.NoError(err)
- singleTester.NotNil(f)
- msg, err := f.Get()
- singleTester.NoError(err)
- singleTester.NotNil(msg)
- // TODO: better error response
- singleTester.NotNil(msg.Data())
- singleTester.Len(msg.Data(), tt.wantLen)
- if tt.checker != nil {
- singleTester.True(tt.checker(msg.Data().([]data.Entity)))
- }
- })
- }
-}
diff --git a/banyand/query/v1/query.go b/banyand/query/v1/query.go
deleted file mode 100644
index 2d2f939..0000000
--- a/banyand/query/v1/query.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package v1
-
-import (
- "context"
-
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-type Executor interface {
- run.PreRunner
-}
-
-func NewExecutor(_ context.Context, serviceRepo discovery.ServiceRepo, indexRepo index.Repo, uniModel series.UniModel,
- schemaRepo series.SchemaRepo, pipeline queue.Queue) (Executor, error) {
- return &queryProcessor{
- Repo: indexRepo,
- UniModel: uniModel,
- schemaRepo: schemaRepo,
- serviceRepo: serviceRepo,
- logger: logger.GetLogger("query"),
- pipeline: pipeline,
- }, nil
-}
diff --git a/banyand/query/v2/processor.go b/banyand/query/v2/processor.go
index 71101cd..d9bfaa3 100644
--- a/banyand/query/v2/processor.go
+++ b/banyand/query/v2/processor.go
@@ -95,5 +95,5 @@ func (q *queryProcessor) Name() string {
func (q *queryProcessor) PreRun() error {
q.log = logger.GetLogger(moduleName)
- return q.pipeline.Subscribe(data.TopicQueryEvent, q)
+ return q.pipeline.Subscribe(data.TopicStreamQuery, q)
}
diff --git a/banyand/query/v2/processor_test.go b/banyand/query/v2/processor_test.go
index 047e97f..542f4d1 100644
--- a/banyand/query/v2/processor_test.go
+++ b/banyand/query/v2/processor_test.go
@@ -70,7 +70,7 @@ var (
}
)
-func setupServices(tester *assert.Assertions) (stream.Service, queue.Queue, func()) {
+func setupServices(tester *require.Assertions) (stream.Service, queue.Queue, func()) {
// Bootstrap logger system
tester.NoError(logger.Init(logger.Logging{
Env: "dev",
@@ -92,7 +92,7 @@ func setupServices(tester *assert.Assertions) (stream.Service, queue.Queue, func
metadataSvc, err := metadata.NewService(context.TODO())
tester.NoError(err)
- streamSvc, err := stream.NewService(context.TODO(), metadataSvc, pipeline)
+ streamSvc, err := stream.NewService(context.TODO(), metadataSvc, repo, pipeline)
tester.NoError(err)
err = streamSvc.FlagSet().Parse([]string{"--root-path=" + rootPath})
@@ -132,12 +132,12 @@ func setupQueryData(testing *testing.T, dataFile string, stream stream.Stream) (
for i, template := range templates {
rawSearchTagFamily, errMarshal := json.Marshal(template)
t.NoError(errMarshal)
- searchTagFamily := &streamv2.ElementValue_TagFamily{}
+ searchTagFamily := &modelv2.TagFamilyForWrite{}
t.NoError(jsonpb.UnmarshalString(string(rawSearchTagFamily), searchTagFamily))
e := &streamv2.ElementValue{
ElementId: strconv.Itoa(i),
Timestamp: timestamppb.New(baseTime.Add(500 * time.Millisecond * time.Duration(i))),
- TagFamilies: []*streamv2.ElementValue_TagFamily{
+ TagFamilies: []*modelv2.TagFamilyForWrite{
{
Tags: []*modelv2.TagValue{
{
@@ -158,7 +158,7 @@ func setupQueryData(testing *testing.T, dataFile string, stream stream.Stream) (
func TestQueryProcessor(t *testing.T) {
assertT := assert.New(t)
- streamSvc, pipeline, deferFunc := setupServices(assertT)
+ streamSvc, pipeline, deferFunc := setupServices(require.New(t))
stm, err := streamSvc.Stream(&commonv2.Metadata{Name: "sw", Group: "default"})
defer func() {
_ = stm.Close()
@@ -335,7 +335,7 @@ func TestQueryProcessor(t *testing.T) {
singleTester := require.New(t)
now := time.Now()
m := bus.NewMessage(bus.MessageID(now.UnixNano()), tt.queryGenerator(baseTs))
- f, err := pipeline.Publish(data.TopicQueryEvent, m)
+ f, err := pipeline.Publish(data.TopicStreamQuery, m)
singleTester.NoError(err)
singleTester.NotNil(f)
msg, err := f.Get()
diff --git a/banyand/series/schema/schema.go b/banyand/series/schema/schema.go
deleted file mode 100644
index d56c63e..0000000
--- a/banyand/series/schema/schema.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package schema
-
-import (
- "context"
-
- "github.com/apache/skywalking-banyandb/api/common"
- apischema "github.com/apache/skywalking-banyandb/api/schema"
-)
-
-type ListOpt struct {
- Group string
-}
-
-type TraceSeries interface {
- Get(ctx context.Context, metadata common.Metadata) (apischema.TraceSeries, error)
- List(ctx context.Context, opt ListOpt) ([]apischema.TraceSeries, error)
-}
-
-type IndexRule interface {
- Get(ctx context.Context, metadata common.Metadata) (apischema.IndexRule, error)
- List(ctx context.Context, opt ListOpt) ([]apischema.IndexRule, error)
-}
-
-type IndexRuleBinding interface {
- Get(ctx context.Context, metadata common.Metadata) (apischema.IndexRuleBinding, error)
- List(ctx context.Context, opt ListOpt) ([]apischema.IndexRuleBinding, error)
-}
diff --git a/banyand/series/schema/sw/index_rule.bin b/banyand/series/schema/sw/index_rule.bin
deleted file mode 100644
index 78fa0e8..0000000
--- a/banyand/series/schema/sw/index_rule.bin
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-default
sw-index-rule
-trace_idtrace_id
-
-service_id
-service_id,
-service_instance_idservice_instance_id
- endpoint_id endpoint_id
-durationduration
- http.method http.method
-db.typedb.type
- db.instance db.instance
-mq.queuemq.queue
-mq.topicmq.topic
- mq.broker mq.broker2���
\ No newline at end of file
diff --git a/banyand/series/schema/sw/index_rule.textproto b/banyand/series/schema/sw/index_rule.textproto
deleted file mode 100644
index bdc932a..0000000
--- a/banyand/series/schema/sw/index_rule.textproto
+++ /dev/null
@@ -1,89 +0,0 @@
-# proto-file: banyandb/v1/schema.proto
-# proto-message: IndexRule
-metadata: {
- name: "sw-index-rule"
- group: "default"
-}
-objects: [
- {
- name: "trace_id",
- fields: [
- "trace_id"
- ],
- type: INDEX_TYPE_ID
- },
- {
- name: "service_id",
- fields: [
- "service_id"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "service_instance_id",
- fields: [
- "service_instance_id"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "endpoint_id",
- fields: [
- "endpoint_id"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "duration",
- fields: [
- "duration"
- ],
- type: INDEX_TYPE_NUMERICAL
- },
- {
- name: "http.method",
- fields: [
- "http.method"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "db.type",
- fields: [
- "db.type"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "db.instance",
- fields: [
- "db.instance"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "mq.queue",
- fields: [
- "mq.queue"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "mq.topic",
- fields: [
- "mq.topic"
- ],
- type: INDEX_TYPE_TEXT
- },
- {
- name: "mq.broker",
- fields: [
- "mq.broker"
- ],
- type: INDEX_TYPE_TEXT
- }
-]
-updated_at: {
- seconds: 1622933202
- nanos: 0
-}
diff --git a/banyand/series/schema/sw/index_rule_binding.bin b/banyand/series/schema/sw/index_rule_binding.bin
deleted file mode 100644
index 8b444f7..0000000
--- a/banyand/series/schema/sw/index_rule_binding.bin
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-defaultsw-index-rule-binding
-default
sw-index-rule
-defaultsw"���*����2���
\ No newline at end of file
diff --git a/banyand/series/schema/sw/index_rule_binding.textproto b/banyand/series/schema/sw/index_rule_binding.textproto
deleted file mode 100644
index 82e9c47..0000000
--- a/banyand/series/schema/sw/index_rule_binding.textproto
+++ /dev/null
@@ -1,31 +0,0 @@
-# proto-file: banyandb/v1/schema.proto
-# proto-message: IndexRuleBinding
-metadata: {
- name: "sw-index-rule-binding"
- group: "default"
-}
-rule_ref: {
- name: "sw-index-rule"
- group: "default"
-}
-subjects: [
- {
- catalog: CATALOG_TRACE
- series: {
- name: "sw"
- group: "default"
- }
- }
-]
-begin_at: {
- seconds: 1622933202
- nanos: 0
-}
-expire_at: {
- seconds: 4778577979
- nanos: 0
-}
-updated_at: {
- seconds: 1622933202
- nanos: 0
-}
diff --git a/banyand/series/schema/sw/sw.go b/banyand/series/schema/sw/sw.go
deleted file mode 100644
index c4d91a0..0000000
--- a/banyand/series/schema/sw/sw.go
+++ /dev/null
@@ -1,126 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//nolint
-//go:generate sh -c "protoc -I../../../../api/proto --encode=banyandb.database.v1.TraceSeries ../../../../api/proto/banyandb/database/v1/schema.proto < trace_series.textproto > trace_series.bin"
-//go:generate sh -c "protoc -I../../../../api/proto --encode=banyandb.database.v1.IndexRule ../../../../api/proto/banyandb/database/v1/schema.proto < index_rule.textproto > index_rule.bin"
-//nolint
-//go:generate sh -c "protoc -I../../../../api/proto --encode=banyandb.database.v1.IndexRuleBinding ../../../../api/proto/banyandb/database/v1/schema.proto < index_rule_binding.textproto > index_rule_binding.bin"
-package sw
-
-import (
- "context"
- //nolint:golint
- _ "embed"
-
- "github.com/golang/protobuf/proto"
-
- "github.com/apache/skywalking-banyandb/api/common"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- apischema "github.com/apache/skywalking-banyandb/api/schema"
- "github.com/apache/skywalking-banyandb/banyand/series/schema"
-)
-
-var (
- _ schema.TraceSeries = (*traceSeriesRepo)(nil)
- _ schema.IndexRule = (*indexRuleRepo)(nil)
-
- //go:embed trace_series.bin
- traceSeriesBin []byte
- //go:embed index_rule.bin
- indexRuleBin []byte
- //go:embed index_rule_binding.bin
- indexRuleBindingBin []byte
-)
-
-type traceSeriesRepo struct {
-}
-
-func NewTraceSeries() schema.TraceSeries {
- return &traceSeriesRepo{}
-}
-
-func (l *traceSeriesRepo) Get(_ context.Context, _ common.Metadata) (apischema.TraceSeries, error) {
- traceSeries := v1.TraceSeries{}
- if err := proto.Unmarshal(traceSeriesBin, &traceSeries); err != nil {
- return apischema.TraceSeries{}, err
- }
- return apischema.TraceSeries{
- KindVersion: apischema.SeriesKindVersion,
- Spec: &traceSeries,
- }, nil
-}
-
-func (l *traceSeriesRepo) List(ctx context.Context, _ schema.ListOpt) ([]apischema.TraceSeries, error) {
- t, err := l.Get(ctx, common.Metadata{})
- if err != nil {
- return nil, err
- }
- return []apischema.TraceSeries{t}, nil
-}
-
-type indexRuleRepo struct {
-}
-
-func NewIndexRule() schema.IndexRule {
- return &indexRuleRepo{}
-}
-
-func (i *indexRuleRepo) Get(ctx context.Context, metadata common.Metadata) (apischema.IndexRule, error) {
- indexRule := v1.IndexRule{}
- if err := proto.Unmarshal(indexRuleBin, &indexRule); err != nil {
- return apischema.IndexRule{}, err
- }
- return apischema.IndexRule{
- KindVersion: apischema.IndexRuleKindVersion,
- Spec: &indexRule,
- }, nil
-}
-
-func (i *indexRuleRepo) List(ctx context.Context, opt schema.ListOpt) ([]apischema.IndexRule, error) {
- t, err := i.Get(ctx, common.Metadata{})
- if err != nil {
- return nil, err
- }
- return []apischema.IndexRule{t}, nil
-}
-
-type indexRuleBindingRepo struct {
-}
-
-func NewIndexRuleBinding() schema.IndexRuleBinding {
- return &indexRuleBindingRepo{}
-}
-
-func (i *indexRuleBindingRepo) Get(_ context.Context, _ common.Metadata) (apischema.IndexRuleBinding, error) {
- indexRuleBinding := v1.IndexRuleBinding{}
- if err := proto.Unmarshal(indexRuleBindingBin, &indexRuleBinding); err != nil {
- return apischema.IndexRuleBinding{}, err
- }
- return apischema.IndexRuleBinding{
- KindVersion: apischema.IndexRuleBindingKindVersion,
- Spec: &indexRuleBinding,
- }, nil
-}
-
-func (i *indexRuleBindingRepo) List(ctx context.Context, _ schema.ListOpt) ([]apischema.IndexRuleBinding, error) {
- t, err := i.Get(ctx, common.Metadata{})
- if err != nil {
- return nil, err
- }
- return []apischema.IndexRuleBinding{t}, nil
-}
diff --git a/banyand/series/schema/sw/trace_series.bin b/banyand/series/schema/sw/trace_series.bin
deleted file mode 100644
index fdccfec..0000000
--- a/banyand/series/schema/sw/trace_series.bin
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
-defaultsw
-trace_id
-state
-
-service_id
-service_instance_id
- endpoint_id
-duration
-
-start_time
- http.method
- status_code
-db.type
- db.instance
-mq.queue
-mq.topic
- mq.broker:
-trace_id
-service_idservice_instance_id
-state01"0
-service_idservice_instance_id endpoint_id*2���
\ No newline at end of file
diff --git a/banyand/series/schema/sw/trace_series.textproto b/banyand/series/schema/sw/trace_series.textproto
deleted file mode 100644
index 663d0a8..0000000
--- a/banyand/series/schema/sw/trace_series.textproto
+++ /dev/null
@@ -1,89 +0,0 @@
-# proto-file: banyandb/v1/schema.proto
-# proto-message: TraceSeries
-metadata: {
- name: "sw"
- group: "default"
-}
-fields: [
- {
- name: "trace_id"
- type: FIELD_TYPE_STRING
- },{
- name: "state"
- type: FIELD_TYPE_INT
- },{
- name: "service_id"
- type: FIELD_TYPE_STRING
- },{
- name: "service_instance_id",
- type: FIELD_TYPE_STRING
- },
- {
- name: "endpoint_id",
- type: FIELD_TYPE_STRING
- },
- {
- name: "duration",
- type: FIELD_TYPE_INT
- },
- {
- name: "start_time",
- type: FIELD_TYPE_INT
- },
- {
- name: "http.method",
- type: FIELD_TYPE_STRING
- },
- {
- name: "status_code",
- type: FIELD_TYPE_STRING
- },
- {
- name: "db.type",
- type: FIELD_TYPE_STRING
- },
- {
- name: "db.instance",
- type: FIELD_TYPE_STRING
- },
- {
- name: "mq.queue",
- type: FIELD_TYPE_STRING
- },
- {
- name: "mq.topic",
- type: FIELD_TYPE_STRING
- },
- {
- name: "mq.broker",
- type: FIELD_TYPE_STRING
- }
-]
-reserved_fields_map: {
- trace_id: "trace_id"
- series_id: [
- "service_id",
- "service_instance_id"
- ]
- state: {
- field: "state",
- val_success: "0",
- val_error: "1"
- }
-}
-shard: {
- number: 2
- routing_fields: [
- "service_id",
- "service_instance_id",
- "endpoint_id"
- ]
-}
-duration: {
- val: 7
- unit: DURATION_UNIT_DAY
-}
-updated_at: {
- seconds: 1622933202
- nanos: 0
-}
\ No newline at end of file
diff --git a/banyand/series/series.go b/banyand/series/series.go
deleted file mode 100644
index d9fdcc1..0000000
--- a/banyand/series/series.go
+++ /dev/null
@@ -1,90 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:generate mockgen -destination=./series_mock.go -package=series . UniModel
-package series
-
-import (
- "context"
- "time"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/series/schema"
- posting2 "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-// TraceState represents the State of a traceSeries link
-type TraceState int
-
-const (
- TraceStateDefault TraceState = iota
- TraceStateSuccess
- TraceStateError
-)
-
-//ScanOptions contain options
-//nolint
-type ScanOptions struct {
- Projection []string
- DataBinary bool
- State TraceState
- Limit uint32
-}
-
-//TraceRepo contains traceSeries and entity data
-type TraceRepo interface {
- //FetchTrace returns data.Trace by traceID
- FetchTrace(traceSeries common.Metadata, traceID string, opt ScanOptions) (data.Trace, error)
- //FetchEntity returns data.Entity by ChunkID
- FetchEntity(traceSeries common.Metadata, shardID uint, chunkIDs posting2.List, opt ScanOptions) ([]data.Entity, error)
- //ScanEntity returns data.Entity between a duration by ScanOptions
- ScanEntity(traceSeries common.Metadata, startTime, endTime uint64, opt ScanOptions) ([]data.Entity, error)
- // Write entity to the given traceSeries
- Write(traceSeries common.Metadata, ts time.Time, seriesID, entityID string, dataBinary []byte, items ...interface{}) (bool, error)
-}
-
-//UniModel combines Trace, Metric and Log repositories into a union interface
-type UniModel interface {
- TraceRepo
-}
-
-//SchemaRepo contains schema definition
-type SchemaRepo interface {
- TraceSeries() schema.TraceSeries
- IndexRule() schema.IndexRule
- IndexRuleBinding() schema.IndexRuleBinding
-}
-
-type IndexObjectFilter func(object *v1.IndexObject) bool
-
-//IndexFilter provides methods to find a specific index related objects
-type IndexFilter interface {
- //IndexRules fetches v1.IndexRule by Series defined in IndexRuleBinding and a filter
- IndexRules(ctx context.Context, subject *v1.Series, filter IndexObjectFilter) ([]*v1.IndexRule, error)
-}
-
-//Service provides operations how to access series module
-type Service interface {
- UniModel
- SchemaRepo
- IndexFilter
- run.PreRunner
- run.Service
-}
diff --git a/banyand/series/trace/common_test.go b/banyand/series/trace/common_test.go
deleted file mode 100644
index f9b2689..0000000
--- a/banyand/series/trace/common_test.go
+++ /dev/null
@@ -1,258 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "context"
- "os"
- "path"
- "sort"
- "strings"
- "testing"
- "time"
-
- "github.com/golang/mock/gomock"
- googleUUID "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/storage"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- v12 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
-
-var _ sort.Interface = (ByEntityID)(nil)
-
-type ByEntityID []data.Entity
-
-func (b ByEntityID) Len() int {
- return len(b)
-}
-
-func (b ByEntityID) Less(i, j int) bool {
- return strings.Compare(b[i].GetEntityId(), b[j].GetEntityId()) < 0
-}
-
-func (b ByEntityID) Swap(i, j int) {
- b[i], b[j] = b[j], b[i]
-}
-
-func setup(t *testing.T) (*traceSeries, func()) {
- _ = logger.Bootstrap()
- db, err := storage.NewDB(context.TODO(), nil)
- assert.NoError(t, err)
- uuid, err := googleUUID.NewUUID()
- assert.NoError(t, err)
- rootPath := path.Join(os.TempDir(), "banyandb-"+uuid.String())
- assert.NoError(t, db.FlagSet().Parse([]string{"--root-path=" + rootPath}))
- ctrl := gomock.NewController(t)
- mockIndex := index.NewMockService(ctrl)
- mockIndex.EXPECT().Insert(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
- svc, err := NewService(context.TODO(), db, nil, mockIndex, nil)
- assert.NoError(t, err)
- assert.NoError(t, svc.PreRun())
- assert.NoError(t, db.PreRun())
- traceSVC := svc.(*service)
- ts, err := traceSVC.getSeries(common.Metadata{
- Spec: &v1.Metadata{
- Name: "sw",
- Group: "default",
- },
- })
- assert.NoError(t, err)
- return ts, func() {
- db.GracefulStop()
- _ = os.RemoveAll(rootPath)
- }
-}
-
-type seriesEntity struct {
- seriesID string
- entity entity
-}
-
-type wantEntity struct {
- entityID string
- dataBinary []byte
- fieldsSize int
-}
-
-var interval = 500 * time.Millisecond
-
-func testData(t time.Time) []seriesEntity {
- return []seriesEntity{
- {
- seriesID: "webapp_10.0.0.1",
- entity: getEntityWithTS("1", []byte{11}, t,
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- ),
- },
- {
- seriesID: "gateway_10.0.0.2",
- entity: getEntityWithTS("2", []byte{12}, t.Add(interval),
- "trace_id-xxfff.111323a",
- 1,
- ),
- },
- {
- seriesID: "httpserver_10.0.0.3",
- entity: getEntityWithTS("3", []byte{13}, t.Add(interval*2),
- "trace_id-xxfff.111323",
- 1,
- "httpserver_id",
- "10.0.0.3_id",
- "/home_id",
- 300,
- 1622933202000000000,
- "GET",
- "200",
- ),
- },
- {
- seriesID: "database_10.0.0.4",
- entity: getEntityWithTS("4", []byte{14}, t.Add(interval*3),
- "trace_id-xxfff.111323",
- 0,
- "database_id",
- "10.0.0.4_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- "MySQL",
- "10.1.1.4",
- ),
- },
- {
- seriesID: "mq_10.0.0.5",
- entity: getEntityWithTS("5", []byte{15}, t.Add(interval*4),
- "trace_id-zzpp.111323",
- 0,
- "mq_id",
- "10.0.0.5_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- nil,
- nil,
- "test_topic",
- "10.0.0.5",
- ),
- },
- {
- seriesID: "database_10.0.0.6",
- entity: getEntityWithTS("6", []byte{16}, t.Add(interval*5),
- "trace_id-zzpp.111323",
- 1,
- "database_id",
- "10.0.0.6_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- "MySQL",
- "10.1.1.6",
- ),
- },
- {
- seriesID: "mq_10.0.0.7",
- entity: getEntityWithTS("7", []byte{17}, t.Add(interval*6),
- "trace_id-zzpp.111323",
- 0,
- "nq_id",
- "10.0.0.7_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- nil,
- nil,
- "test_topic",
- "10.0.0.7",
- ),
- },
- }
-}
-
-type entity struct {
- id string
- binary []byte
- t time.Time
- items []interface{}
-}
-
-func getEntity(id string, binary []byte, items ...interface{}) entity {
- return entity{
- id: id,
- binary: binary,
- items: items,
- }
-}
-
-func getEntityWithTS(id string, binary []byte, t time.Time, items ...interface{}) entity {
- return entity{
- id: id,
- binary: binary,
- t: t,
- items: items,
- }
-}
-
-func setupTestData(t *testing.T, ts *traceSeries, seriesEntities []seriesEntity) (results []idWithShard) {
- results = make([]idWithShard, 0, len(seriesEntities))
- for _, se := range seriesEntities {
- seriesID := []byte(se.seriesID)
- ev := v12.NewEntityValueBuilder().
- DataBinary(se.entity.binary).
- EntityID(se.entity.id).
- Timestamp(se.entity.t).
- Fields(se.entity.items...).
- Build()
- shardID, _ := partition.ShardID(seriesID, 2)
- got, err := ts.Write(common.SeriesID(convert.Hash(seriesID)), shardID, data.EntityValue{
- EntityValue: ev,
- })
- if err != nil {
- t.Error("write() got error")
- }
- if got < 1 {
- t.Error("write() got empty chunkID")
- }
- results = append(results, idWithShard{
- id: got,
- shardID: shardID,
- })
- }
- return results
-}
diff --git a/banyand/series/trace/query.go b/banyand/series/trace/query.go
deleted file mode 100644
index 522532f..0000000
--- a/banyand/series/trace/query.go
+++ /dev/null
@@ -1,279 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "encoding/hex"
- "time"
-
- "github.com/golang/protobuf/proto"
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- v12 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- "github.com/apache/skywalking-banyandb/pkg/posting"
- "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
-)
-
-func (t *traceSeries) FetchTrace(traceID string, opt series.ScanOptions) (trace data.Trace, err error) {
- if traceID == "" {
- return trace, ErrInvalidTraceID
- }
- traceIDBytes := []byte(traceID)
- traceIDShardID, shardIDError := partition.ShardID(traceIDBytes, t.shardNum)
- if shardIDError != nil {
- return trace, shardIDError
- }
- bb, errTraceID := t.reader.TimeSeriesReader(traceIDShardID, traceIndex, 0, 0).GetAll(traceIDBytes)
- if errTraceID != nil {
- return trace, errTraceID
- }
- t.l.Debug().Uint("shard_id", traceIDShardID).
- Str("trace_id", traceID).
- Hex("trace_id_bytes", traceIDBytes).
- Int("chunk_num", len(bb)).Msg("fetch Trace by trace_id")
- if len(bb) < 1 {
- return trace, nil
- }
- dataMap := make(map[uint]posting.List)
- for _, b := range bb {
- id := idWithShard{
- id: common.ChunkID(convert.BytesToUint64(b[2:])),
- shardID: uint(convert.BytesToUint16(b[:2])),
- }
- placeID(dataMap, id)
- }
- var entities []data.Entity
- for s, c := range dataMap {
- ee, errEntity := t.FetchEntity(c, s, opt)
- if errEntity != nil {
- err = multierr.Append(err, errEntity)
- continue
- }
- entities = append(entities, ee...)
- }
- return data.Trace{
- KindVersion: data.TraceKindVersion,
- Entities: entities,
- }, err
-}
-
-func (t *traceSeries) ScanEntity(startTime, endTime uint64, opt series.ScanOptions) ([]data.Entity, error) {
- total := opt.Limit
- if total < 1 {
- total = 10
- }
- states := make([]byte, 0, 2)
- switch opt.State {
- case series.TraceStateSuccess:
- states = append(states, StateSuccess)
- case series.TraceStateError:
- states = append(states, StateError)
- case series.TraceStateDefault:
- states = append(states, StateSuccess, StateError)
- }
- seekKeys := make([][]byte, 0, len(states))
- startTimeBytes := convert.Uint64ToBytes(startTime)
- for _, state := range states {
- key := make([]byte, 8+1)
- key[0] = state
- copy(key[1:], startTimeBytes)
- seekKeys = append(seekKeys, key)
- }
- entities := make([]data.Entity, 0, total)
- var num uint32
- opts := kv.DefaultScanOpts
- opts.PrefetchValues = false
- opts.PrefetchSize = int(total)
- var errAll error
- for i := uint(0); i < uint(t.shardNum); i++ {
- chunkIDs := roaring.NewPostingList()
- for _, seekKey := range seekKeys {
- state := seekKey[0]
- err := t.reader.Reader(i, startTimeIndex, startTime, endTime).Scan(
- seekKey,
- opts,
- func(shardID int, key []byte, _ func() ([]byte, error)) error {
- if len(key) <= 9 {
- return errors.Wrapf(ErrInvalidKey, "key:%s", hex.EncodeToString(key))
- }
- if key[0] != state {
- return kv.ErrStopScan
- }
- ts := convert.BytesToUint64(key[1 : 8+1])
- if ts > endTime {
- return nil
- }
- chunk := make([]byte, len(key)-8-1)
- copy(chunk, key[8+1:])
- chunkIDs.Insert(common.ChunkID(convert.BytesToUint64(chunk)))
- num++
- if num > total {
- return kv.ErrStopScan
- }
- return nil
- })
- if err != nil {
- errAll = multierr.Append(errAll, err)
- }
- }
- if chunkIDs.IsEmpty() {
- continue
- }
- ee, err := t.FetchEntity(chunkIDs, i, opt)
- if err != nil {
- errAll = multierr.Append(errAll, err)
- continue
- }
- entities = append(entities, ee...)
- }
- return entities, errAll
-}
-
-func (t *traceSeries) FetchEntity(chunkIDs posting.List, shardID uint, opt series.ScanOptions) (entities []data.Entity, err error) {
- chunkIDsLen := chunkIDs.Len()
- if chunkIDsLen < 1 {
- return nil, ErrChunkIDsEmpty
- }
- entities = make([]data.Entity, 0, chunkIDsLen)
- fetchFieldsIndices, errInfo := t.parseFetchInfo(opt)
- fetchDataBinary := opt.DataBinary
- if errInfo != nil {
- return nil, errInfo
- }
- if !fetchDataBinary && len(fetchFieldsIndices) < 1 {
- return nil, ErrProjectionEmpty
- }
-
- for iter := chunkIDs.Iterator(); iter.Next(); {
- id := iter.Current()
- chunkID := uint64(id)
- ts, errParseTS := t.idGen.ParseTS(chunkID)
- if errParseTS != nil {
- err = multierr.Append(err, errParseTS)
- }
- ref, chunkErr := t.reader.Reader(shardID, chunkIDMapping, ts, ts).Get(convert.Uint64ToBytes(chunkID))
- if chunkErr != nil {
- err = multierr.Append(err, chunkErr)
- continue
- }
- sRef := ref[:len(ref)-8]
- seriesID := sRef[1:]
- state := sRef[0]
-
- t.l.Debug().
- Uint64("chunk_id", chunkID).
- Hex("id", ref).
- Uint64("series_id", convert.BytesToUint64(seriesID)).
- Uint("shard_id", shardID).
- Time("ts", time.Unix(0, int64(ts))).
- Uint64("ts_int", ts).
- Msg("fetch internal id by chunk_id")
- entity, errGet := t.getEntityByInternalRef(seriesID, State(state), fetchDataBinary, fetchFieldsIndices, shardID, ts)
- if errGet != nil {
- err = multierr.Append(err, errGet)
- continue
- }
- t.l.Debug().
- Str("entity_id", entity.GetEntityId()).
- Int("fields_num", len(entity.GetFields())).
- Int("data_binary_size_bytes", len(entity.GetDataBinary())).
- Msg("fetch entity")
- entities = append(entities, entity)
- }
- return entities, err
-}
-
-func (t *traceSeries) parseFetchInfo(opt series.ScanOptions) (fetchFieldsIndices []v12.FieldEntry, err error) {
- fetchFieldsIndices = make([]v12.FieldEntry, 0)
- for _, p := range opt.Projection {
- f, ok := t.fieldIndex[p]
- if !ok {
- return nil, errors.Wrapf(ErrFieldNotFound, "field name:%s", p)
- }
- fetchFieldsIndices = append(fetchFieldsIndices, v12.FieldEntry{
- Key: p,
- Index: f.idx,
- Type: f.spec.GetType(),
- })
- t.l.Debug().Str("name", p).Interface("index", f).Msg("to fetch the field")
- }
- return fetchFieldsIndices, nil
-}
-
-func (t *traceSeries) getEntityByInternalRef(seriesID []byte, state State, fetchDataBinary bool,
- fetchFieldsIndices []v12.FieldEntry, shardID uint, ts uint64) (data.Entity, error) {
- fieldsStore, dataStore, err := getStoreName(state)
- if err != nil {
- return data.Entity{}, err
- }
- val, getErr := t.reader.TimeSeriesReader(shardID, fieldsStore, ts, ts).Get(seriesID, ts)
- if getErr != nil {
- return data.Entity{}, getErr
- }
- // deserialize write.EntityValue
- entityVal := &v1.EntityValue{}
- if err := proto.Unmarshal(val, entityVal); err != nil {
- return data.Entity{}, err
- }
- // transform to query.Entity
- entity := v1.Entity{
- EntityId: entityVal.GetEntityId(),
- Timestamp: entityVal.GetTimestamp(),
- }
-
- // Copy selected fields
- if len(fetchFieldsIndices) > 0 {
- entity.Fields = v12.Transform(entityVal, fetchFieldsIndices)
- }
-
- if fetchDataBinary {
- val, getErr = t.reader.TimeSeriesReader(shardID, dataStore, ts, ts).Get(seriesID, ts)
- if getErr != nil {
- return data.Entity{}, getErr
- }
- entity.DataBinary = val
- }
-
- return data.Entity{
- Entity: &entity,
- }, nil
-}
-
-type idWithShard struct {
- id common.ChunkID
- shardID uint
-}
-
-func placeID(chunkIDCriteria map[uint]posting.List, data idWithShard) {
- list, ok := chunkIDCriteria[data.shardID]
- if ok {
- list.Insert(data.id)
- return
- }
- list = roaring.NewPostingList()
- list.Insert(data.id)
- chunkIDCriteria[data.shardID] = list
-}
diff --git a/banyand/series/trace/query_test.go b/banyand/series/trace/query_test.go
deleted file mode 100644
index 251c607..0000000
--- a/banyand/series/trace/query_test.go
+++ /dev/null
@@ -1,321 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "math"
- "sort"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/pkg/posting"
-)
-
-func Test_traceSeries_FetchEntity(t *testing.T) {
- type args struct {
- chunkIDIndices []int
- chunkIDs []idWithShard
- opt series.ScanOptions
- }
- tests := []struct {
- name string
- args args
- wantEntities []wantEntity
- wantErr bool
- }{
- {
- name: "golden path",
- args: args{
- chunkIDIndices: []int{0, 1, 2, 3, 4, 5, 6},
- opt: series.ScanOptions{DataBinary: true, Projection: []string{"trace_id"}},
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- {entityID: "2", dataBinary: []byte{12}, fieldsSize: 1},
- {entityID: "3", dataBinary: []byte{13}, fieldsSize: 1},
- {entityID: "4", dataBinary: []byte{14}, fieldsSize: 1},
- {entityID: "5", dataBinary: []byte{15}, fieldsSize: 1},
- {entityID: "6", dataBinary: []byte{16}, fieldsSize: 1},
- {entityID: "7", dataBinary: []byte{17}, fieldsSize: 1},
- },
- },
- {
- name: "multiple fields",
- args: args{
- chunkIDIndices: []int{0, 1, 2, 3, 4, 5, 6},
- opt: series.ScanOptions{Projection: []string{"trace_id", "service_id", "state", "duration", "mq.queue"}},
- },
- wantEntities: []wantEntity{
- {entityID: "1", fieldsSize: 5},
- {entityID: "2", fieldsSize: 5},
- {entityID: "3", fieldsSize: 5},
- {entityID: "4", fieldsSize: 5},
- {entityID: "5", fieldsSize: 5},
- {entityID: "6", fieldsSize: 5},
- {entityID: "7", fieldsSize: 5},
- },
- },
- {
- name: "data binary",
- args: args{
- chunkIDIndices: []int{0, 1, 2, 3, 4, 5, 6},
- opt: series.ScanOptions{DataBinary: true, Projection: []string{}},
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}},
- {entityID: "2", dataBinary: []byte{12}},
- {entityID: "3", dataBinary: []byte{13}},
- {entityID: "4", dataBinary: []byte{14}},
- {entityID: "5", dataBinary: []byte{15}},
- {entityID: "6", dataBinary: []byte{16}},
- {entityID: "7", dataBinary: []byte{17}},
- },
- },
- {
- name: "invalid chunk ids",
- args: args{
- chunkIDs: []idWithShard{
- {
- id: common.ChunkID(0),
- },
- },
- opt: series.ScanOptions{DataBinary: true, Projection: []string{"trace_id"}},
- },
- wantErr: true,
- },
- {
- name: "mix up invalid/valid ids",
- args: args{
- chunkIDs: []idWithShard{
- {
- id: common.ChunkID(0),
- },
- },
- chunkIDIndices: []int{0, 1},
- opt: series.ScanOptions{DataBinary: true, Projection: []string{"trace_id"}},
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- {entityID: "2", dataBinary: []byte{12}, fieldsSize: 1},
- },
- wantErr: true,
- },
- {
- name: "absent scan opt",
- args: args{
- chunkIDIndices: []int{0, 1},
- },
- wantErr: true,
- },
- {
- name: "invalid opt absent",
- args: args{
- chunkIDIndices: []int{0, 1},
- opt: series.ScanOptions{Projection: []string{"trace_id", "undefined"}},
- },
- wantErr: true,
- },
- }
- ts, stopFunc := setup(t)
- defer stopFunc()
- dataResult := setupTestData(t, ts, testData(time.Now()))
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- chunkIDCriteria := make(map[uint]posting.List, 2)
- for i := range tt.args.chunkIDIndices {
- placeID(chunkIDCriteria, dataResult[i])
- }
- for _, id := range tt.args.chunkIDs {
- placeID(chunkIDCriteria, id)
- }
- var entities ByEntityID
- var err error
- for s, c := range chunkIDCriteria {
- ee, errFetch := ts.FetchEntity(c, s, tt.args.opt)
- if errFetch != nil {
- err = multierr.Append(err, errFetch)
- }
- entities = append(entities, ee...)
- }
- if (err != nil) != tt.wantErr {
- t.Errorf("write() error = %v, wantErr %v", err, tt.wantErr)
- }
- sort.Sort(entities)
- assert.Equal(t, len(tt.wantEntities), len(entities))
- for i, e := range entities {
- assert.EqualValues(t, tt.wantEntities[i].entityID, e.GetEntityId())
- assert.Equal(t, tt.wantEntities[i].dataBinary, e.GetDataBinary())
- assert.Len(t, e.GetFields(), tt.wantEntities[i].fieldsSize)
- }
- })
- }
-}
-
-func Test_traceSeries_FetchTrace(t *testing.T) {
- type args struct {
- traceID string
- }
- tests := []struct {
- name string
- args args
- wantEntities []wantEntity
- wantErr bool
- }{
- {
- name: "golden path",
- args: args{
- traceID: "trace_id-xxfff.111323",
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- {entityID: "2", dataBinary: []byte{12}, fieldsSize: 1},
- {entityID: "3", dataBinary: []byte{13}, fieldsSize: 1},
- {entityID: "4", dataBinary: []byte{14}, fieldsSize: 1},
- },
- },
- {
- name: "found nothing",
- args: args{
- traceID: "not_existed",
- },
- },
- {
- name: "absent Trace id",
- args: args{},
- wantErr: true,
- },
- }
- ts, stopFunc := setup(t)
- defer stopFunc()
- setupTestData(t, ts, testData(time.Now()))
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- traceData, err := ts.FetchTrace(tt.args.traceID, series.ScanOptions{DataBinary: true, Projection: []string{"trace_id"}})
- if (err != nil) != tt.wantErr {
- t.Errorf("write() error = %v, wantErr %v", err, tt.wantErr)
- }
- var entities ByEntityID = traceData.Entities
- assert.Equal(t, len(tt.wantEntities), len(entities))
- sort.Sort(entities)
- for i, e := range entities {
- assert.EqualValues(t, tt.wantEntities[i].entityID, e.GetEntityId())
- assert.Equal(t, tt.wantEntities[i].dataBinary, e.GetDataBinary())
- assert.Len(t, e.GetFields(), tt.wantEntities[i].fieldsSize)
- }
- })
- }
-}
-
-func Test_traceSeries_ScanEntity(t *testing.T) {
- type args struct {
- start time.Time
- end time.Time
- }
- baseTS := time.Now()
- tests := []struct {
- name string
- args args
- wantEntities []wantEntity
- wantErr bool
- }{
- {
- name: "scan all",
- args: args{
- start: time.Unix(0, 0),
- end: time.Unix(math.MaxInt64, math.MaxInt64),
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- {entityID: "2", dataBinary: []byte{12}, fieldsSize: 1},
- {entityID: "3", dataBinary: []byte{13}, fieldsSize: 1},
- {entityID: "4", dataBinary: []byte{14}, fieldsSize: 1},
- {entityID: "5", dataBinary: []byte{15}, fieldsSize: 1},
- {entityID: "6", dataBinary: []byte{16}, fieldsSize: 1},
- {entityID: "7", dataBinary: []byte{17}, fieldsSize: 1},
- },
- },
- {
- name: "scan range",
- args: args{
- start: baseTS.Add(-interval * 1),
- end: baseTS.Add(interval * 6),
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- {entityID: "2", dataBinary: []byte{12}, fieldsSize: 1},
- {entityID: "3", dataBinary: []byte{13}, fieldsSize: 1},
- {entityID: "4", dataBinary: []byte{14}, fieldsSize: 1},
- {entityID: "5", dataBinary: []byte{15}, fieldsSize: 1},
- {entityID: "6", dataBinary: []byte{16}, fieldsSize: 1},
- {entityID: "7", dataBinary: []byte{17}, fieldsSize: 1},
- },
- },
- {
- name: "scan slice",
- args: args{
- start: baseTS.Add(interval + time.Millisecond),
- end: baseTS.Add(5*interval - 2*time.Millisecond),
- },
- wantEntities: []wantEntity{
- {entityID: "3", dataBinary: []byte{13}, fieldsSize: 1},
- {entityID: "4", dataBinary: []byte{14}, fieldsSize: 1},
- {entityID: "5", dataBinary: []byte{15}, fieldsSize: 1},
- },
- },
- {
- name: "single result",
- args: args{
- start: time.Unix(0, 0),
- end: baseTS,
- },
- wantEntities: []wantEntity{
- {entityID: "1", dataBinary: []byte{11}, fieldsSize: 1},
- },
- },
- {
- name: "found nothing",
- args: args{},
- },
- }
- ts, stopFunc := setup(t)
- defer stopFunc()
- setupTestData(t, ts, testData(baseTS))
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- var entities ByEntityID
- var err error
- entities, err = ts.ScanEntity(uint64(tt.args.start.UnixNano()), uint64(tt.args.end.UnixNano()), series.ScanOptions{DataBinary: true, Projection: []string{"trace_id"}})
- if (err != nil) != tt.wantErr {
- t.Errorf("write() error = %v, wantErr %v", err, tt.wantErr)
- }
- assert.Equal(t, len(tt.wantEntities), len(entities))
- sort.Sort(entities)
- for i, e := range entities {
- assert.GreaterOrEqual(t, tt.args.end.UnixNano(), e.Timestamp.AsTime().UnixNano())
- assert.Equal(t, tt.wantEntities[i].entityID, e.GetEntityId())
- assert.Equal(t, tt.wantEntities[i].dataBinary, e.GetDataBinary())
- assert.Len(t, e.GetFields(), tt.wantEntities[i].fieldsSize)
- }
- })
- }
-}
diff --git a/banyand/series/trace/schema.go b/banyand/series/trace/schema.go
deleted file mode 100644
index f98d039..0000000
--- a/banyand/series/trace/schema.go
+++ /dev/null
@@ -1,102 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "context"
- "time"
-
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/banyand/series/schema"
- "github.com/apache/skywalking-banyandb/banyand/series/schema/sw"
-)
-
-//Methods for query objects in the schema
-
-func (s *service) TraceSeries() schema.TraceSeries {
- return sw.NewTraceSeries()
-}
-
-func (s *service) IndexRule() schema.IndexRule {
- return sw.NewIndexRule()
-}
-
-func (s *service) IndexRuleBinding() schema.IndexRuleBinding {
- return sw.NewIndexRuleBinding()
-}
-
-func (s *service) IndexRules(ctx context.Context, subject *databasev1.Series, filter series.IndexObjectFilter) ([]*databasev1.IndexRule, error) {
- group := subject.Series.GetGroup()
- bindings, err := s.IndexRuleBinding().List(ctx, schema.ListOpt{Group: group})
- if err != nil {
- return nil, err
- }
- subjectSeries := subject.GetSeries()
- if subjectSeries == nil {
- return nil, nil
- }
- now := time.Now()
- foundRules := make([]*commonv1.Metadata, 0)
- for _, binding := range bindings {
- spec := binding.Spec
- if spec.GetBeginAt().AsTime().After(now) ||
- spec.GetExpireAt().AsTime().Before(now) {
- continue
- }
- for _, sub := range spec.GetSubjects() {
- if sub != nil && sub.GetCatalog() == subject.GetCatalog() {
- s1 := sub.GetSeries()
- if s1 != nil &&
- s1.GetName() == subjectSeries.GetName() &&
- s1.GetGroup() == subjectSeries.GetGroup() {
- ruleRef := spec.GetRuleRef()
- if ruleRef != nil {
- foundRules = append(foundRules, ruleRef)
- }
- }
- break
- }
- }
- }
- result := make([]*databasev1.IndexRule, 0)
- var indexRuleErr error
- for _, rule := range foundRules {
- object, getErr := s.IndexRule().Get(ctx, common.Metadata{KindVersion: common.MetadataKindVersion, Spec: rule})
- if getErr != nil {
- indexRuleErr = multierr.Append(indexRuleErr, err)
- continue
- }
- r := object.Spec
- if filter == nil {
- result = append(result, r)
- continue
- }
- for _, obj := range r.GetObjects() {
- if filter(obj) {
- result = append(result, r)
- continue
- }
- }
- }
- return result, indexRuleErr
-}
diff --git a/banyand/series/trace/schema_test.go b/banyand/series/trace/schema_test.go
deleted file mode 100644
index adcbef8..0000000
--- a/banyand/series/trace/schema_test.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "context"
- "reflect"
- "testing"
-
- "github.com/apache/skywalking-banyandb/api/common"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/banyand/series/schema/sw"
-)
-
-func Test_service_RulesBySubject(t *testing.T) {
- type args struct {
- series *databasev1.Series
- filter series.IndexObjectFilter
- }
- tests := []struct {
- name string
- args args
- want []*databasev1.IndexRule
- wantErr bool
- }{
- {
- name: "golden path",
- args: args{
- series: createSubject("sw", "default"),
- },
- want: getIndexRule("sw-index-rule", "default"),
- },
- {
- name: "filter index object",
- args: args{
- series: createSubject("sw", "default"),
- filter: func(object *databasev1.IndexObject) bool {
- return object.GetFields()[0] == "trace_id"
- },
- },
- want: getIndexRule("sw-index-rule", "default"),
- },
- {
- name: "got empty idWithShard",
- args: args{
- series: createSubject("sw", "default"),
- filter: func(object *databasev1.IndexObject) bool {
- return object.GetFields()[0] == "invalid"
- },
- },
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := &service{}
- ctx := context.Background()
- got, err := s.IndexRules(ctx, tt.args.series, tt.args.filter)
- if (err != nil) != tt.wantErr {
- t.Errorf("RulesBySubject() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if len(got) < 1 && len(tt.want) < 1 {
- return
- }
- if !reflect.DeepEqual(got, tt.want) {
- t.Errorf("RulesBySubject() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
-func getIndexRule(name, group string) []*databasev1.IndexRule {
- indexRule, _ := sw.NewIndexRule().Get(context.Background(), common.Metadata{
- KindVersion: common.MetadataKindVersion,
- Spec: &commonv1.Metadata{
- Group: group,
- Name: name,
- }},
- )
- return []*databasev1.IndexRule{indexRule.Spec}
-}
-
-func createSubject(name, group string) *databasev1.Series {
- return &databasev1.Series{
- Series: &commonv1.Metadata{
- Group: group,
- Name: name,
- },
- Catalog: databasev1.Series_CATALOG_TRACE,
- }
-}
diff --git a/banyand/series/trace/service.go b/banyand/series/trace/service.go
deleted file mode 100644
index 72f73b0..0000000
--- a/banyand/series/trace/service.go
+++ /dev/null
@@ -1,186 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "context"
- "time"
-
- "google.golang.org/protobuf/types/known/timestamppb"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- "github.com/apache/skywalking-banyandb/api/event"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/queue"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/banyand/series/schema"
- "github.com/apache/skywalking-banyandb/banyand/storage"
- "github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
-
-var _ series.Service = (*service)(nil)
-
-type service struct {
- db storage.Database
- schemaMap map[string]*traceSeries
- l *logger.Logger
- repo discovery.ServiceRepo
- stopCh chan struct{}
- idx index.Service
- writeListener *writeCallback
- pipeline queue.Queue
-}
-
-//NewService returns a new service
-func NewService(_ context.Context, db storage.Database, repo discovery.ServiceRepo, idx index.Service, pipeline queue.Queue) (series.Service, error) {
- return &service{
- db: db,
- repo: repo,
- idx: idx,
- pipeline: pipeline,
- writeListener: &writeCallback{},
- }, nil
-}
-
-func (s *service) Name() string {
- return "trace-series"
-}
-
-func (s *service) PreRun() error {
- schemas, err := s.TraceSeries().List(context.Background(), schema.ListOpt{})
- if err != nil {
- return err
- }
- s.schemaMap = make(map[string]*traceSeries, len(schemas))
- s.writeListener.schemaMap = s.schemaMap
- s.l = logger.GetLogger(s.Name())
- s.writeListener.l = s.l
- for _, sa := range schemas {
- ts, errTS := newTraceSeries(sa, s.l, s.idx)
- if errTS != nil {
- return errTS
- }
- s.db.Register(ts)
- id := formatTraceSeriesID(ts.name, ts.group)
- s.schemaMap[id] = ts
- s.writeListener.schemaMap[id] = ts
- s.l.Info().Str("id", id).Msg("initialize Trace series")
- }
- return err
-}
-
-func (s *service) Serve() error {
- now := time.Now().UnixNano()
- for _, sMeta := range s.schemaMap {
- e := v1.NewSeriesEventBuilder().
- SeriesMetadata(sMeta.group, sMeta.name).
- FieldNames(sMeta.fieldsNamesCompositeSeriesID...).
- Time(time.Now()).
- Action(databasev1.Action_ACTION_PUT).
- Build()
- _, err := s.repo.Publish(event.TopicSeriesEvent, bus.NewMessage(bus.MessageID(now), e))
- if err != nil {
- return err
- }
- seriesObj := &databasev1.Series{
- Series: &commonv1.Metadata{
- Name: sMeta.name,
- Group: sMeta.group,
- },
- Catalog: databasev1.Series_CATALOG_TRACE,
- }
- rules, errGetRules := s.IndexRules(context.Background(), seriesObj, nil)
- if errGetRules != nil {
- return errGetRules
- }
- shardedRuleIndex := make([]*databasev1.IndexRuleEvent_ShardedIndexRule, 0, len(rules)*int(sMeta.shardNum))
- for i := 0; i < int(sMeta.shardNum); i++ {
- t := time.Now()
- e := v1.NewShardEventBuilder().Action(databasev1.Action_ACTION_PUT).Time(t).
- Shard(
- v1.NewShardBuilder().
- ID(uint64(i)).Total(sMeta.shardNum).SeriesMetadata(sMeta.group, sMeta.name).UpdatedAt(t).CreatedAt(t).
- Node(v1.NewNodeBuilder().
- ID(s.repo.NodeID()).CreatedAt(t).UpdatedAt(t).Addr("localhost").
- Build()).
- Build()).
- Build()
- _, errShard := s.repo.Publish(event.TopicShardEvent, bus.NewMessage(bus.MessageID(now), e))
- if errShard != nil {
- return errShard
- }
- shardedRuleIndex = append(shardedRuleIndex, &databasev1.IndexRuleEvent_ShardedIndexRule{
- ShardId: uint64(i),
- Rules: rules,
- })
- }
-
- indexRule := &databasev1.IndexRuleEvent{
- Series: seriesObj.Series,
- Rules: shardedRuleIndex,
- Action: databasev1.Action_ACTION_PUT,
- Time: timestamppb.New(time.Now()),
- }
- _, errPublishRules := s.repo.Publish(event.TopicIndexRule, bus.NewMessage(bus.MessageID(now), indexRule))
- if errPublishRules != nil {
- return errPublishRules
- }
- }
- errWrite := s.pipeline.Subscribe(data.TopicWriteEvent, s.writeListener)
- if errWrite != nil {
- return errWrite
- }
- s.stopCh = make(chan struct{})
- <-s.stopCh
- return nil
-}
-
-func (s *service) GracefulStop() {
- if s.stopCh != nil {
- close(s.stopCh)
- }
-}
-
-type writeCallback struct {
- l *logger.Logger
- schemaMap map[string]*traceSeries
-}
-
-func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
- writeEvent, ok := message.Data().(data.TraceWriteDate)
- if !ok {
- w.l.Warn().Msg("invalid event data type")
- return
- }
- entityValue := writeEvent.WriteRequest.GetEntity()
- ts := writeEvent.WriteRequest.GetMetadata()
- id := formatTraceSeriesID(ts.GetName(), ts.GetGroup())
- _, err := w.schemaMap[id].Write(common.SeriesID(writeEvent.SeriesID), writeEvent.ShardID, data.EntityValue{
- EntityValue: entityValue,
- })
- if err != nil {
- w.l.Warn().Err(err)
- }
- return
-}
diff --git a/banyand/series/trace/trace.go b/banyand/series/trace/trace.go
deleted file mode 100644
index 2cf79a1..0000000
--- a/banyand/series/trace/trace.go
+++ /dev/null
@@ -1,377 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "strconv"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- apischema "github.com/apache/skywalking-banyandb/api/schema"
- "github.com/apache/skywalking-banyandb/banyand/index"
- "github.com/apache/skywalking-banyandb/banyand/series"
- "github.com/apache/skywalking-banyandb/banyand/storage"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
- posting2 "github.com/apache/skywalking-banyandb/pkg/posting"
-)
-
-const (
- // KV stores
- chunkIDMapping = "chunkIDMapping"
- startTimeIndex = "startTimeIndex"
- // Time series stores
- successDataStore = "successDataStore"
- successFieldsStore = "successFieldsStore"
- errorDataStore = "errorDataStore"
- errorFieldsStore = "errorFieldsStore"
-
- traceIndex = "traceIndex"
-)
-
-var (
- _ storage.Plugin = (*traceSeries)(nil)
- _ series.TraceRepo = (*service)(nil)
- ErrTraceSeriesNotFound = errors.New("failed to find Trace series")
- ErrFieldSchemaNotFound = errors.New("failed to find trace_id specification")
- ErrFieldNotFound = errors.New("failed to find specific field")
- ErrProjectionEmpty = errors.New("projection is empty")
- ErrChunkIDsEmpty = errors.New("chunkID is empty")
- ErrInvalidTraceID = errors.New("invalid Trace id")
- ErrUnsupportedFieldType = errors.New("unsupported field type")
- ErrUnknownFieldValue = errors.New("unknown field value")
- ErrInvalidKey = errors.New("invalid key")
- ErrUnknownState = errors.New("unknown state value")
-)
-
-type State byte
-
-const (
- StateSuccess = 0
- StateError = 1
-)
-
-func (s *service) FetchTrace(traceSeries common.Metadata, traceID string, opt series.ScanOptions) (data.Trace, error) {
- ts, err := s.getSeries(traceSeries)
- if err != nil {
- return data.Trace{}, err
- }
- return ts.FetchTrace(traceID, opt)
-}
-
-func (s *service) FetchEntity(traceSeries common.Metadata, shardID uint, chunkIDs posting2.List, opt series.ScanOptions) ([]data.Entity, error) {
- ts, err := s.getSeries(traceSeries)
- if err != nil {
- return nil, err
- }
- return ts.FetchEntity(chunkIDs, shardID, opt)
-}
-
-func (s *service) ScanEntity(traceSeries common.Metadata, startTime, endTime uint64, opt series.ScanOptions) ([]data.Entity, error) {
- ts, err := s.getSeries(traceSeries)
- if err != nil {
- return nil, err
- }
- return ts.ScanEntity(startTime, endTime, opt)
-}
-
-func (s *service) getSeries(traceSeries common.Metadata) (*traceSeries, error) {
- id := formatTraceSeriesID(traceSeries.Spec.GetName(), traceSeries.Spec.GetGroup())
- s.l.Debug().Str("id", id).Msg("got Trace series")
- ts, ok := s.schemaMap[id]
- if !ok {
- return nil, errors.Wrapf(ErrTraceSeriesNotFound, "series id:%s, map:%v", id, s.schemaMap)
- }
- return ts, nil
-}
-
-func (s *service) Write(traceSeriesMetadata common.Metadata, ts time.Time, seriesID, entityID string, dataBinary []byte, items ...interface{}) (bool, error) {
- traceSeries, err := s.getSeries(traceSeriesMetadata)
- if err != nil {
- return false, err
- }
-
- ev := v1.NewEntityValueBuilder().
- DataBinary(dataBinary).
- EntityID(entityID).
- Fields(items...).
- Timestamp(ts).
- Build()
-
- seriesIDBytes := []byte(seriesID)
- shardID, shardIDError := partition.ShardID(seriesIDBytes, traceSeries.shardNum)
- if shardIDError != nil {
- return err == nil, shardIDError
- }
- _, err = traceSeries.Write(common.SeriesID(convert.Hash(seriesIDBytes)), shardID, data.EntityValue{
- EntityValue: ev,
- })
- return err == nil, err
-}
-
-func formatTraceSeriesID(name, group string) string {
- return name + ":" + group
-}
-
-type traceSeries struct {
- name string
- group string
- idGen series.IDGen
- l *logger.Logger
- schema apischema.TraceSeries
- reader storage.StoreRepo
- writePoint storage.GetWritePoint
- idx index.Service
- shardNum uint32
- fieldIndex map[string]*fieldSpec
- traceIDIndex int
- traceIDFieldName string
- stateFieldName string
- stateFieldType databasev1.FieldType
- strStateSuccessVal string
- strStateErrorVal string
- intStateSuccessVal int64
- intStateErrorVal int64
- stateIndex int
- fieldsNamesCompositeSeriesID []string
-}
-
-type fieldSpec struct {
- idx int
- spec *databasev1.FieldSpec
-}
-
-func newTraceSeries(schema apischema.TraceSeries, l *logger.Logger, idx index.Service) (*traceSeries, error) {
- t := &traceSeries{
- schema: schema,
- idGen: series.NewIDGen(),
- l: l,
- idx: idx,
- }
- meta := t.schema.Spec.GetMetadata()
- shardInfo := t.schema.Spec.GetShard()
- t.shardNum = shardInfo.GetNumber()
- t.name, t.group = meta.GetName(), meta.GetGroup()
- if err := t.buildFieldIndex(); err != nil {
- return nil, err
- }
- traceID, ok := t.fieldIndex[t.traceIDFieldName]
- if !ok {
- return nil, errors.Wrapf(ErrFieldSchemaNotFound, "trace_id field name:%s\n field index:%v",
- t.traceIDFieldName, t.fieldIndex)
- }
- t.traceIDIndex = traceID.idx
- state, ok := t.fieldIndex[t.stateFieldName]
- if !ok {
- return nil, errors.Wrapf(ErrFieldSchemaNotFound, "state field name:%s\n field index:%v",
- t.traceIDFieldName, t.fieldIndex)
- }
- t.stateIndex = state.idx
- return t, nil
-}
-
-func (t *traceSeries) Meta() storage.PluginMeta {
- return storage.PluginMeta{
- ID: t.name,
- Group: t.group,
- ShardNumber: t.shardNum,
- KVSpecs: []storage.KVSpec{
- {
- Name: chunkIDMapping,
- Type: storage.KVTypeNormal,
- },
- {
- Name: startTimeIndex,
- Type: storage.KVTypeNormal,
- },
-
- {
- Name: successDataStore,
- Type: storage.KVTypeTimeSeries,
- CompressLevel: 3,
- },
- {
- Name: successFieldsStore,
- Type: storage.KVTypeTimeSeries,
- CompressLevel: 3,
- },
- {
- Name: errorDataStore,
- Type: storage.KVTypeTimeSeries,
- CompressLevel: 3,
- },
- {
- Name: errorFieldsStore,
- Type: storage.KVTypeTimeSeries,
- CompressLevel: 3,
- },
- {
- Name: traceIndex,
- Type: storage.KVTypeTimeSeries,
- CompressLevel: -1,
- },
- },
- }
-}
-
-func (t *traceSeries) Init(repo storage.StoreRepo, point storage.GetWritePoint) {
- t.reader = repo
- t.writePoint = point
-}
-
-func (t *traceSeries) buildFieldIndex() error {
- spec := t.schema.Spec
- reservedMap := spec.GetReservedFieldsMap()
- t.traceIDFieldName = reservedMap.GetTraceId()
- state := reservedMap.GetState()
- stateFieldName := state.GetField()
-
- fieldsLen := len(spec.GetFields())
- t.fieldIndex = make(map[string]*fieldSpec, fieldsLen)
- for idx, f := range spec.GetFields() {
- if f.GetName() == stateFieldName {
- t.stateFieldType = f.GetType()
- }
- t.fieldIndex[f.GetName()] = &fieldSpec{
- idx: idx,
- spec: f,
- }
- }
- switch t.stateFieldType {
- case databasev1.FieldType_FIELD_TYPE_STRING:
- t.strStateSuccessVal = state.GetValSuccess()
- t.strStateErrorVal = state.GetValError()
- case databasev1.FieldType_FIELD_TYPE_INT:
- intSVal, err := strconv.ParseInt(state.GetValSuccess(), 10, 64)
- if err != nil {
- return err
- }
- t.intStateSuccessVal = intSVal
- intEVal, err := strconv.ParseInt(state.GetValError(), 10, 64)
- if err != nil {
- return err
- }
- t.intStateErrorVal = intEVal
- default:
- return errors.Wrapf(ErrUnsupportedFieldType, "type:%s, supported type: Int and String", t.stateFieldType.String())
- }
- t.stateFieldName = stateFieldName
-
- t.fieldsNamesCompositeSeriesID = make([]string, 0, len(reservedMap.GetSeriesId()))
- for i := 0; i < len(reservedMap.GetSeriesId()); i++ {
- t.fieldsNamesCompositeSeriesID = append(t.fieldsNamesCompositeSeriesID, reservedMap.GetSeriesId()[i])
- }
-
- return nil
-}
-
-// getTraceID extracts traceID as bytes from v1.EntityValue
-func (t *traceSeries) getTraceID(entityValue *tracev1.EntityValue) ([]byte, error) {
- if entityValue.GetFields() == nil {
- return nil, errors.Wrapf(ErrFieldNotFound, "EntityValue does not contain any fields")
- }
- if len(entityValue.GetFields()) < t.traceIDIndex+1 {
- return nil, errors.Wrapf(ErrFieldNotFound, "EntityValue contains incomplete fields")
- }
- f := entityValue.GetFields()[t.traceIDIndex]
- if f == nil {
- return nil, errors.Wrapf(ErrFieldNotFound, "trace_id index %d", t.traceIDIndex)
- }
- switch v := f.GetValueType().(type) {
- case *modelv1.Field_Str:
- return []byte(v.Str.GetValue()), nil
- default:
- // TODO: add a test to cover the default case
- return nil, errors.Wrapf(ErrUnsupportedFieldType, "type: %v, supported type: String", v)
- }
-}
-
-func (t *traceSeries) getState(entityValue *tracev1.EntityValue) (state State, fieldStoreName, dataStoreName string, err error) {
- if entityValue.GetFields() == nil {
- err = errors.Wrapf(ErrFieldNotFound, "EntityValue does not contain any fields")
- return
- }
- if len(entityValue.GetFields()) < t.stateIndex+1 {
- err = errors.Wrapf(ErrFieldNotFound, "EntityValue contains incomplete fields")
- return
- }
-
- f := entityValue.GetFields()[t.stateIndex]
- if f == nil {
- err = errors.Wrapf(ErrFieldNotFound, "state index %d", t.stateIndex)
- return
- }
-
- switch v := f.GetValueType().(type) {
- case *modelv1.Field_Int:
- if t.stateFieldType != databasev1.FieldType_FIELD_TYPE_INT {
- // TODO: add a test case to cover this line
- err = errors.Wrapf(ErrUnsupportedFieldType, "given type: Int, supported type: %s", t.stateFieldType.String())
- return
- }
- switch v.Int.GetValue() {
- case t.intStateSuccessVal:
- state = StateSuccess
- case t.intStateErrorVal:
- state = StateError
- default:
- err = errors.Wrapf(ErrUnknownFieldValue, "value:%d, supported value: %d, %d",
- v.Int.GetValue(), t.intStateSuccessVal, t.intStateErrorVal)
- return
- }
- case *modelv1.Field_Str:
- if t.stateFieldType != databasev1.FieldType_FIELD_TYPE_STRING {
- err = errors.Wrapf(ErrUnsupportedFieldType, "given type: String, supported type: %s", t.stateFieldType.String())
- return
- }
- switch v.Str.GetValue() {
- case t.strStateSuccessVal:
- state = StateSuccess
- case t.strStateErrorVal:
- state = StateError
- default:
- err = errors.Wrapf(ErrUnknownFieldValue, "value:%s, supported value: %s, %s",
- v.Str.GetValue(), t.strStateSuccessVal, t.strStateErrorVal)
- return
- }
- default:
- // TODO: cover?
- err = errors.Wrapf(ErrUnsupportedFieldType, "type: %s, supported type: String and Int", v)
- return
- }
- fieldStoreName, dataStoreName, err = getStoreName(state)
- return
-}
-
-func getStoreName(state State) (string, string, error) {
- switch state {
- case StateSuccess:
- return successFieldsStore, successDataStore, nil
- case StateError:
- return errorFieldsStore, errorDataStore, nil
- }
- return "", "", ErrUnknownState
-}
diff --git a/banyand/series/trace/write.go b/banyand/series/trace/write.go
deleted file mode 100644
index 87d202a..0000000
--- a/banyand/series/trace/write.go
+++ /dev/null
@@ -1,157 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "time"
-
- "github.com/golang/protobuf/proto"
- "github.com/pkg/errors"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/banyand/index"
- bydb_bytes "github.com/apache/skywalking-banyandb/pkg/bytes"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/partition"
-)
-
-//TODO: To add WAL to handle crashing
-//TODO: To replace timestamp with monotonic ID in case of server time reset
-func (t *traceSeries) Write(seriesID common.SeriesID, shardID uint, entity data.EntityValue) (common.ChunkID, error) {
- entityVal := entity.EntityValue
- traceID, err := t.getTraceID(entityVal)
- if err != nil {
- return 0, err
- }
- state, fieldsStoreName, dataStoreName, errGetState := t.getState(entityVal)
- if errGetState != nil {
- return 0, errGetState
- }
- stateBytes := []byte{byte(state)}
- entityTs := uint64(entity.GetTimestamp().AsTime().UnixNano())
- chunkID := t.idGen.Next(entityTs)
- wallTs, errParseTS := t.idGen.ParseTS(chunkID)
- if errParseTS != nil {
- return 0, errors.Wrap(errParseTS, "failed to parse timestamp from chunk id")
- }
- wallTsBytes := convert.Uint64ToBytes(wallTs)
- intSeriesID := uint64(seriesID)
- seriesIDBytes := convert.Uint64ToBytes(intSeriesID)
- wp := t.writePoint(wallTs)
-
- err = wp.TimeSeriesWriter(shardID, dataStoreName).Put(seriesIDBytes, entityVal.GetDataBinary(), wallTs)
- if err != nil {
- return 0, errors.Wrap(err, "fail to write traceSeries data")
- }
-
- byteVal, err := proto.Marshal(copyEntityValueWithoutDataBinary(entityVal))
- if err != nil {
- return 0, errors.Wrap(err, "fail to serialize EntityValue to []byte")
- }
- err = wp.TimeSeriesWriter(shardID, fieldsStoreName).Put(seriesIDBytes, byteVal, wallTs)
- if err != nil {
- return 0, errors.Wrap(err, "failed to write traceSeries fields")
- }
-
- chunkIDBytes := convert.Uint64ToBytes(chunkID)
- if err = wp.Writer(shardID, chunkIDMapping).Put(chunkIDBytes, bydb_bytes.Join(stateBytes, seriesIDBytes, wallTsBytes)); err != nil {
- return 0, errors.Wrap(err, "failed to write chunkID index")
- }
- traceIDShardID, shardIDError := partition.ShardID(traceID, t.shardNum)
- if shardIDError != nil {
- return 0, shardIDError
- }
- if err = wp.TimeSeriesWriter(traceIDShardID, traceIndex).
- Put(traceID, bydb_bytes.Join(convert.Uint16ToBytes(uint16(shardID)), chunkIDBytes), entityTs); err != nil {
- return 0, errors.Wrap(err, "failed to Trace index")
- }
- entityTsBytes := convert.Uint64ToBytes(entityTs)
- err = wp.Writer(shardID, startTimeIndex).Put(bydb_bytes.Join(stateBytes, entityTsBytes, chunkIDBytes), nil)
- if err != nil {
- return 0, errors.Wrap(err, "failed to write start time index")
- }
- t.l.Debug().Uint64("chunk_id", chunkID).
- Uint64("series_id", intSeriesID).
- Time("wallTs", time.Unix(0, int64(wallTs))).
- Uint64("wallTs_int", wallTs).
- Int("data_size", len(entityVal.GetDataBinary())).
- Int("fields_num", len(entityVal.GetFields())).
- Hex("trace_id", traceID).
- Uint("trace_shard_id", traceIDShardID).
- Uint("shard_id", shardID).
- Msg("written to Trace series")
- id := common.ChunkID(chunkID)
- for i, field := range entityVal.GetFields() {
- fieldSpec := t.schema.Spec.GetFields()[i]
- fieldName := fieldSpec.GetName()
- switch x := field.ValueType.(type) {
- case *modelv1.Field_Str:
- err = multierr.Append(err, t.writeStrToIndex(shardID, id, fieldName, x.Str.GetValue()))
- case *modelv1.Field_Int:
- err = multierr.Append(err, t.writeIntToIndex(shardID, id, fieldName, x.Int.GetValue()))
- case *modelv1.Field_StrArray:
- for _, s := range x.StrArray.GetValue() {
- err = multierr.Append(err, t.writeStrToIndex(shardID, id, fieldName, s))
- }
- case *modelv1.Field_IntArray:
- for _, integer := range x.IntArray.GetValue() {
- err = multierr.Append(err, t.writeIntToIndex(shardID, id, fieldName, integer))
- }
- default:
- continue
- }
- }
- return common.ChunkID(chunkID), err
-}
-
-func (t *traceSeries) writeIntToIndex(shardID uint, id common.ChunkID, name string, value int64) error {
- return t.writeIndex(shardID, id, name, convert.Int64ToBytes(value))
-}
-
-func (t *traceSeries) writeStrToIndex(shardID uint, id common.ChunkID, name string, value string) error {
- return t.writeIndex(shardID, id, name, []byte(value))
-}
-
-func (t *traceSeries) writeIndex(shardID uint, id common.ChunkID, name string, value []byte) error {
- return t.idx.Insert(*common.NewMetadata(&commonv1.Metadata{
- Name: t.name,
- Group: t.group,
- }),
- shardID,
- &index.Field{
- ChunkID: id,
- Name: name,
- Value: value,
- },
- )
-}
-
-// copyEntityValueWithoutDataBinary copies all fields without DataBinary
-func copyEntityValueWithoutDataBinary(ev *tracev1.EntityValue) *tracev1.EntityValue {
- return &tracev1.EntityValue{
- EntityId: ev.GetEntityId(),
- Timestamp: ev.GetTimestamp(),
- DataBinary: nil,
- Fields: ev.GetFields(),
- }
-}
diff --git a/banyand/series/trace/write_test.go b/banyand/series/trace/write_test.go
deleted file mode 100644
index e2a1872..0000000
--- a/banyand/series/trace/write_test.go
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package trace
-
-import (
- "testing"
- "time"
-
- "github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- "github.com/apache/skywalking-banyandb/pkg/convert"
- "github.com/apache/skywalking-banyandb/pkg/partition"
- v1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
-)
-
-func Test_traceSeries_Write(t *testing.T) {
- ts, stopFunc := setup(t)
- defer stopFunc()
-
- tests := []struct {
- name string
- args seriesEntity
- wantErr bool
- }{
- {
- name: "golden path",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- ),
- },
- },
- {
- name: "minimal",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 1,
- ),
- },
- },
- {
- name: "http",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- "GET",
- "200",
- ),
- },
- },
- {
- name: "database",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- "MySQL",
- "10.1.1.2",
- ),
- },
- },
- {
- name: "mq",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 1,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- nil,
- nil,
- nil,
- nil,
- "test_topic",
- "10.0.0.1",
- ),
- },
- },
- {
- name: "absent Trace id",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- nil,
- 0,
- ),
- },
- wantErr: true,
- },
- {
- name: "invalid Trace id",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- 1212323,
- 1,
- ),
- },
- wantErr: true,
- },
- {
- name: "absent State",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- nil,
- ),
- },
- wantErr: true,
- },
- {
- name: "invalid State",
- args: seriesEntity{
- seriesID: "webapp_10.0.0.1",
- entity: getEntity("1231.dfd.123123ssf", []byte{12},
- "trace_id-xxfff.111323",
- 6,
- ),
- },
- wantErr: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- seriesID := []byte(tt.args.seriesID)
- shardID, shardIDError := partition.ShardID(seriesID, 2)
- if shardIDError != nil {
- return
- }
- ev := v1.NewEntityValueBuilder().
- DataBinary(tt.args.entity.binary).
- EntityID(tt.args.entity.id).
- Fields(tt.args.entity.items...).
- Timestamp(time.Now()).
- Build()
- got, err := ts.Write(common.SeriesID(convert.Hash(seriesID)), shardID, data.EntityValue{
- EntityValue: ev,
- })
- if (err != nil) != tt.wantErr {
- t.Errorf("write() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if err == nil && got < 1 {
- t.Error("write() got empty chunkID")
- }
- })
- }
-}
diff --git a/banyand/storage/block.go b/banyand/storage/block.go
deleted file mode 100644
index 76a8b1f..0000000
--- a/banyand/storage/block.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package storage
-
-import (
- "fmt"
-
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-type block struct {
- path string
- plugin Plugin
-
- l *logger.Logger
-
- stores map[string]kv.Store
- tsStores map[string]kv.TimeSeriesStore
- indexStores map[string]kv.IndexStore
-
- shardID int
-}
-
-func newBlock(shardID int, path string, plugin Plugin) (*block, error) {
- l := logger.GetLogger("block")
- return &block{
- shardID: shardID,
- path: path,
- plugin: plugin,
- l: l,
- stores: make(map[string]kv.Store),
- tsStores: make(map[string]kv.TimeSeriesStore),
- indexStores: make(map[string]kv.IndexStore),
- }, nil
-}
-
-func (b *block) init() (err error) {
- meta := b.plugin.Meta()
- kvDefines := meta.KVSpecs
- if kvDefines == nil || len(kvDefines) < 1 {
- return nil
- }
- if err = b.createKV(kvDefines); err != nil {
- return err
- }
- return nil
-}
-
-func (b *block) createKV(defines []KVSpec) (err error) {
- for _, define := range defines {
- storeID := define.Name
- path := fmt.Sprintf("%s/%s", b.path, storeID)
- b.l.Info().Str("path", path).Uint8("type", uint8(define.Type)).Msg("open kv store")
- switch define.Type {
- case KVTypeNormal:
- var s kv.Store
- opts := make([]kv.StoreOptions, 0)
- if define.BufferSize > 0 {
- opts = append(opts, kv.StoreWithBufferSize(define.BufferSize))
- }
- if define.FlushCallback != nil {
- opts = append(opts, kv.StoreWithFlushCallback(define.FlushCallback))
- }
- opts = append(opts, kv.StoreWithLogger(b.l))
- if s, err = kv.OpenStore(b.shardID, path, opts...); err != nil {
- return fmt.Errorf("failed to open normal store: %w", err)
- }
- b.stores[storeID] = s
- case KVTypeTimeSeries:
- var s kv.TimeSeriesStore
- if s, err = kv.OpenTimeSeriesStore(b.shardID, path, define.CompressLevel, define.ValueSize,
- kv.TSSWithLogger(b.l)); err != nil {
- return fmt.Errorf("failed to open time series store: %w", err)
- }
- b.tsStores[storeID] = s
- case KVTypeIndex:
- var s kv.IndexStore
- if s, err = kv.OpenIndexStore(b.shardID, path, kv.IndexWithLogger(b.l)); err != nil {
- return fmt.Errorf("failed to open time series store: %w", err)
- }
- b.indexStores[storeID] = s
- }
- }
- return nil
-}
-
-func (b *block) close() {
- for _, store := range b.stores {
- _ = store.Close()
- }
- for _, store := range b.tsStores {
- _ = store.Close()
- }
- for _, store := range b.indexStores {
- _ = store.Close()
- }
-}
diff --git a/banyand/storage/database.go b/banyand/storage/database.go
deleted file mode 100644
index 9302ab0..0000000
--- a/banyand/storage/database.go
+++ /dev/null
@@ -1,321 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package storage
-
-import (
- "errors"
- "fmt"
- "io/fs"
- "io/ioutil"
- "os"
- "sync"
- "time"
-
- "go.uber.org/atomic"
- "go.uber.org/multierr"
-
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-const (
- shardTemplate = "%s/shard-%d"
- segTemplate = "%s/seg-%s"
- blockTemplate = "%s/block-%s"
-
- segFormat = "20060102"
- blockFormat = "1504"
-
- dirPerm = 0700
-)
-
-var ErrNoDiscoveryRepo = errors.New("no discovery repo exists")
-
-var _ Database = (*DB)(nil)
-
-// DB is a storage manager to physical data model.
-// Notice: The current status of DB is under WIP. It only contains feats support to verify the series module.
-type DB struct {
- root string
- seriesLst []*series
- repo discovery.ServiceRepo
- pluginLst []Plugin
-
- stopCh chan struct{}
- log *logger.Logger
-}
-
-func (d *DB) Serve() error {
- d.stopCh = make(chan struct{})
- <-d.stopCh
- return nil
-}
-
-func (d *DB) GracefulStop() {
- for _, s := range d.seriesLst {
- s.stop()
- }
- if d.stopCh != nil {
- close(d.stopCh)
- }
-}
-
-func (d *DB) Register(plugin Plugin) {
- d.pluginLst = append(d.pluginLst, plugin)
-}
-
-func (d *DB) Name() string {
- return "database"
-}
-
-func (d *DB) FlagSet() *run.FlagSet {
- flagS := run.NewFlagSet("storage")
- flagS.StringVar(&d.root, "root-path", "/tmp", "the root path of database")
- return flagS
-}
-
-func (d *DB) Validate() error {
- return nil
-}
-
-func (d *DB) PreRun() error {
- d.log = logger.GetLogger("database")
- if err := d.init(); err != nil {
- return fmt.Errorf("failed to initialize db: %v", err)
- }
- return nil
-}
-
-type series struct {
- sLst []*shard
- init bool
- location string
-}
-
-func (s *series) Reader(shard uint, name string, start, end uint64) kv.Reader {
- //TODO: find targets in all blocks
- b, ok := s.sLst[shard].activeBlock.Load().(*block)
- if !ok {
- return nil
- }
- return b.stores[name]
-}
-
-func (s *series) TimeSeriesReader(shard uint, name string, start, end uint64) kv.TimeSeriesReader {
- //TODO: find targets in all blocks
- b, ok := s.sLst[shard].activeBlock.Load().(*block)
- if !ok {
- return nil
- }
- return b.tsStores[name]
-}
-
-func (s *series) Index(shard uint, name string) kv.IndexStore {
- //TODO: find targets in all blocks
- b, ok := s.sLst[shard].activeBlock.Load().(*block)
- if !ok {
- return nil
- }
- return b.indexStores[name]
-}
-
-func (s *series) load(meta PluginMeta) error {
- //TODO: to implement load instead of removing old contents
- return os.RemoveAll(s.location)
-}
-
-func (s *series) create(meta PluginMeta, plugin Plugin) (err error) {
- if _, err = mkdir(s.location); err != nil {
- return err
- }
-
- for i := 0; i < int(meta.ShardNumber); i++ {
- var shardLocation string
- if shardLocation, err = mkdir(shardTemplate, s.location, i); err != nil {
- return err
- }
- so := newShard(i, shardLocation)
- if sErr := so.init(plugin); sErr != nil {
- err = multierr.Append(err, sErr)
- continue
- }
- s.sLst = append(s.sLst, so)
- }
- return err
-
-}
-
-func (s *series) stop() {
- for _, sa := range s.sLst {
- sa.stop()
- }
-}
-
-func (d *DB) init() (err error) {
- if _, err = mkdir(d.root); err != nil {
- return fmt.Errorf("failed to create %s: %v", d.root, err)
- }
- d.log.Info().Str("path", d.root).Msg("initialize database")
- var entries []fs.FileInfo
- if entries, err = ioutil.ReadDir(d.root); err != nil {
- return fmt.Errorf("failed to read directory contents failed: %v", err)
- }
- for _, plugin := range d.pluginLst {
- meta := plugin.Meta()
-
- s := &series{location: d.root + "/" + meta.ID}
- d.seriesLst = append(d.seriesLst, s)
- for _, entry := range entries {
- if entry.Name() == meta.ID && entry.IsDir() {
- err = s.load(meta)
- if err != nil {
- return err
- }
- d.log.Info().Str("ID", meta.ID).Msg("loaded series")
- break
- }
- }
- if !s.init {
- err = s.create(meta, plugin)
- if err != nil {
- return err
- }
- d.log.Info().Str("ID", meta.ID).Msg("created series")
- }
- plugin.Init(s, func(_ uint64) WritePoint {
- bLst := make([]*block, len(s.sLst))
- for i, sa := range s.sLst {
- b, ok := sa.activeBlock.Load().(*block)
- if !ok {
- continue
- }
- bLst[i] = b
- }
- return &writePoint{bLst: bLst}
- })
- d.log.Info().Str("ID", meta.ID).Msg("initialized plugin")
- }
- return err
-}
-
-var _ WritePoint = (*writePoint)(nil)
-
-type writePoint struct {
- bLst []*block
-}
-
-func (w *writePoint) Close() error {
- return nil
-}
-
-func (w *writePoint) Writer(shard uint, name string) kv.Writer {
- return w.bLst[shard].stores[name]
-}
-
-func (w *writePoint) TimeSeriesWriter(shard uint, name string) kv.TimeSeriesWriter {
- return w.bLst[shard].tsStores[name]
-}
-
-type shard struct {
- id int
- lst []*segment
- sync.Mutex
- location string
- activeBlock atomic.Value
-}
-
-func newShard(id int, location string) *shard {
- return &shard{
- id: id,
- location: location,
- }
-}
-
-func (s *shard) newSeg(shardID int, path string) *segment {
- s.Lock()
- defer s.Unlock()
- seg := &segment{
- path: path,
- shardID: shardID,
- }
- s.lst = append(s.lst, seg)
- return seg
-}
-
-func (s *shard) init(plugin Plugin) error {
- segPath, err := mkdir(segTemplate, s.location, time.Now().Format(segFormat))
- if err != nil {
- return fmt.Errorf("failed to make segment directory: %v", err)
- }
-
- seg := s.newSeg(s.id, segPath)
- return seg.init(plugin, s.updateActiveBlock)
-}
-
-func (s *shard) updateActiveBlock(newBlock *block) {
- s.activeBlock.Store(newBlock)
-}
-
-func (s *shard) stop() {
- for _, seg := range s.lst {
- seg.close()
- }
-}
-
-func mkdir(format string, a ...interface{}) (path string, err error) {
- path = fmt.Sprintf(format, a...)
- if err = os.MkdirAll(path, dirPerm); err != nil {
- return "", err
- }
- return path, err
-}
-
-type segment struct {
- lst []*block
- sync.Mutex
- path string
- shardID int
-}
-
-func (s *segment) addBlock(b *block) {
- s.Lock()
- defer s.Unlock()
- s.lst = append(s.lst, b)
-}
-
-func (s *segment) init(plugin Plugin, activeBlock func(newBlock *block)) error {
- blockPath, err := mkdir(blockTemplate, s.path, time.Now().Format(blockFormat))
- if err != nil {
- return fmt.Errorf("failed to make block directory: %v", err)
- }
- var b *block
- if b, err = newBlock(s.shardID, blockPath, plugin); err != nil {
- return fmt.Errorf("failed to create segment: %v", err)
- }
- activeBlock(b)
- s.addBlock(b)
- return b.init()
-}
-
-func (s *segment) close() {
- for _, b := range s.lst {
- b.close()
- }
-}
diff --git a/banyand/storage/database_test.go b/banyand/storage/database_test.go
deleted file mode 100644
index 42893e0..0000000
--- a/banyand/storage/database_test.go
+++ /dev/null
@@ -1,277 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package storage
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "os"
- "testing"
- "time"
-
- "github.com/golang/mock/gomock"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-func TestDB_Create_Directory(t *testing.T) {
- ctrl := gomock.NewController(t)
- p := NewMockPlugin(ctrl)
- p.EXPECT().Meta().Return(PluginMeta{
- ID: "sw",
- Group: "default",
- ShardNumber: 1,
- KVSpecs: []KVSpec{
- {
- Name: "test",
- Type: KVTypeTimeSeries,
- CompressLevel: 3,
- },
- },
- }).AnyTimes()
- p.EXPECT().Init(gomock.Any(), gomock.Any()).AnyTimes()
- tempDir, _ := setUp(t, p)
- defer removeDir(tempDir)
- shardPath := fmt.Sprintf(shardTemplate, tempDir+"/sw", 0)
- validateDirectory(t, shardPath)
- now := time.Now()
- segPath := fmt.Sprintf(segTemplate, shardPath, now.Format(segFormat))
- validateDirectory(t, segPath)
- validateDirectory(t, fmt.Sprintf(blockTemplate, segPath, now.Format(blockFormat)))
-}
-
-//
-//func TestDB_Store(t *testing.T) {
-// is := require.New(t)
-// ctrl := gomock.NewController(t)
-// defer ctrl.Finish()
-// now := uint64(time.Now().UnixNano())
-// var ap WritePoint
-// var repo StoreRepo
-// p := mockPlugin(ctrl, func(r StoreRepo, get GetWritePoint) {
-// ap = get(now)
-// repo = r
-// })
-//
-// tempDir, db := setUp(t, p)
-// defer func() {
-// db.GracefulStop()
-// removeDir(tempDir)
-// }()
-//
-// is.NoError(ap.Writer(0, "normal").Put([]byte("key1"), []byte{12}))
-// val, err := repo.Reader(0, "normal", now, now).Get([]byte("key1"))
-// is.NoError(err)
-// is.Equal([]byte{12}, val)
-//
-// is.NoError(ap.TimeSeriesWriter(1, "time-series").Put([]byte("key11"), []byte{33}, 1))
-// val, err = repo.TimeSeriesReader(1, "time-series", now, now).Get([]byte("key11"), 1)
-// is.NoError(err)
-// is.Equal([]byte{33}, val)
-// vals, allErr := repo.TimeSeriesReader(1, "time-series", now, now).GetAll([]byte("key11"))
-// is.NoError(allErr)
-// is.Equal([][]byte{{33}}, vals)
-//
-// index := repo.Index(1, "index")
-// is.NoError(index.Handover(mockMemtable([]uint64{1, 2}, []uint64{3, 6})))
-// list, err := index.Seek(convert.Int64ToBytes(0), 2)
-// is.NoError(err)
-// is.Equal(2, list.Len())
-// is.True(list.Contains(common.ChunkID(1)))
-// is.True(list.Contains(common.ChunkID(2)))
-// list, err = index.Seek(convert.Int64ToBytes(1), 2)
-// is.NoError(err)
-// is.Equal(2, list.Len())
-// is.True(list.Contains(common.ChunkID(3)))
-// is.True(list.Contains(common.ChunkID(6)))
-//
-// is.NoError(index.Handover(mockMemtable([]uint64{11, 14})))
-// list, err = index.Seek(convert.Int64ToBytes(0), 2)
-// is.NoError(err)
-// is.Equal(4, list.Len())
-// is.True(list.Contains(common.ChunkID(1)))
-// is.True(list.Contains(common.ChunkID(2)))
-// is.True(list.Contains(common.ChunkID(11)))
-// is.True(list.Contains(common.ChunkID(14)))
-//}
-
-func TestDB_FlushCallback(t *testing.T) {
- is := require.New(t)
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- now := uint64(time.Now().UnixNano())
- var ap WritePoint
- //var repo StoreRepo
- p := NewMockPlugin(ctrl)
- latch := make(chan struct{})
- closed := false
- p.EXPECT().Meta().Return(PluginMeta{
- ID: "sw",
- Group: "default",
- ShardNumber: 2,
- KVSpecs: []KVSpec{
- {
- Name: "normal",
- Type: KVTypeNormal,
- BufferSize: 10 << 20,
- FlushCallback: func() {
- if closed {
- return
- }
- close(latch)
- closed = true
- },
- },
- },
- }).AnyTimes()
- p.EXPECT().Init(gomock.Any(), gomock.Any()).Do(func(r StoreRepo, wp GetWritePoint) {
- ap = wp(now)
- }).AnyTimes()
-
- tempDir, db := setUp(t, p)
- defer func() {
- db.GracefulStop()
- removeDir(tempDir)
- }()
- for i := 0; i < 5000; i++ {
- key := make([]byte, i)
- _ = ap.Writer(0, "normal").Put(key, []byte{1})
- }
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
- select {
- case <-latch:
- case <-ctx.Done():
- is.Fail("timeout")
- }
-}
-
-//
-//var _ kv.Iterator2 = (*iter)(nil)
-//
-//type iter struct {
-// data map[int]posting.List
-// p int
-//}
-//
-//func (i *iter) Next() {
-// i.p++
-//}
-//
-//func (i *iter) Rewind() {
-// i.p = 0
-//}
-//
-//func (i *iter) Seek(key []byte) {
-// panic("implement me")
-//}
-//
-//func (i *iter) Key() []byte {
-// return convert.Int64ToBytes(int64(i.p))
-//}
-//
-//func (i *iter) Val() posting.List {
-// return i.data[i.p]
-//}
-//
-//func (i *iter) Valid() bool {
-// _, ok := i.data[i.p]
-// return ok
-//}
-//
-//func (i *iter) Close() error {
-// return nil
-//}
-//
-//func mockMemtable(data ...[]uint64) kv.Iterator2 {
-// it := &iter{
-// data: make(map[int]posting.List),
-// }
-// for i, d := range data {
-// it.data[i] = roaring.NewPostingListWithInitialData(d...)
-// }
-// return it
-//}
-//
-//func mockPlugin(ctrl *gomock.Controller, f func(repo StoreRepo, get GetWritePoint)) Plugin {
-// p := NewMockPlugin(ctrl)
-// p.EXPECT().Meta().Return(PluginMeta{
-// ID: "sw",
-// Group: "default",
-// ShardNumber: 2,
-// KVSpecs: []KVSpec{
-// {
-// Name: "normal",
-// Type: KVTypeNormal,
-// },
-// {
-// Name: "time-series",
-// Type: KVTypeTimeSeries,
-// CompressLevel: 3,
-// },
-// {
-// Name: "index",
-// Type: KVTypeIndex,
-// },
-// },
-// }).AnyTimes()
-// p.EXPECT().Init(gomock.Any(), gomock.Any()).Do(func(r StoreRepo, wp GetWritePoint) {
-// f(r, wp)
-// }).AnyTimes()
-// return p
-//}
-
-func setUp(t *testing.T, p Plugin) (tempDir string, db Database) {
- require.NoError(t, logger.Init(logger.Logging{
- Env: "dev",
- Level: "debug",
- }))
- db, err := NewDB(context.Background(), nil)
- require.NoError(t, err)
- require.NotNil(t, db)
-
- var tempDirErr error
- tempDir, tempDirErr = ioutil.TempDir("", "banyandb-test-*")
- require.Nil(t, tempDirErr)
-
- require.NoError(t, db.FlagSet().Parse([]string{"--root-path", tempDir}))
- if p != nil {
- db.Register(p)
- }
- require.NoError(t, db.PreRun())
- go func() {
- require.NoError(t, db.Serve())
- }()
- return tempDir, db
-}
-
-func validateDirectory(t *testing.T, dir string) {
- info, err := os.Stat(dir)
- assert.False(t, os.IsNotExist(err), "Directory does not exist: %v", dir)
- assert.NoError(t, err, "Directory error: %v", dir)
- assert.True(t, info.IsDir(), "Directory is a file, not a directory: %#v\n", dir)
-}
-
-func removeDir(dir string) {
- if err := os.RemoveAll(dir); err != nil {
- fmt.Printf("Error while removing dir: %v\n", err)
- }
-}
diff --git a/banyand/storage/storage.go b/banyand/storage/storage.go
deleted file mode 100644
index 6219842..0000000
--- a/banyand/storage/storage.go
+++ /dev/null
@@ -1,110 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:generate mockgen -destination=./storage_mock.go -package=storage . Plugin
-package storage
-
-import (
- "context"
- "io"
-
- "github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/kv"
- "github.com/apache/skywalking-banyandb/pkg/run"
-)
-
-// KVType defines the kind of a KV storage
-type KVType uint8
-
-const (
- // KVTypeNormal is normal KV storage
- KVTypeNormal KVType = 0
- // KVTypeTimeSeries is a time-series KV storage
- KVTypeTimeSeries KVType = 1
- // KVTypeIndex is an index KV storage
- KVTypeIndex KVType = 2
-)
-
-// Database is the storage manager which implements the physical data model
-type Database interface {
- run.Config
- run.PreRunner
- run.Service
- PluginRepo
-}
-
-type PluginRepo interface {
- // Register a Plugin into a Database
- Register(plugin Plugin)
-}
-
-// NewDB returns a new Database
-func NewDB(_ context.Context, repo discovery.ServiceRepo) (Database, error) {
- return &DB{repo: repo}, nil
-}
-
-// Plugin helps Database create kv storage with a specific data model.
-// Trace, metric and log series could be registered as Plugin to save data into kv regardless of where the kv
-// are placed on the physical devices.
-type Plugin interface {
- Meta() PluginMeta
- // Init the Plugin, and generate KVSpec to indicate how kv storages are built
- Init(StoreRepo, GetWritePoint)
-}
-
-type PluginMeta struct {
- ID string
- Group string
- ShardNumber uint32
- KVSpecs []KVSpec
-}
-
-type CompressSpec struct {
- //Level closest matches the compression ratio of a specific zstd compression level.
- Level int
- Size int
-}
-
-// KVSpec defines the behaviours of a KV store.
-// Database get these specs which are generated by Plugin to create underlying kv storage.
-type KVSpec struct {
- Name string
- Type KVType
- BufferSize int64
- CompressLevel int
- ValueSize int
- FlushCallback kv.FlushCallback
-}
-
-type StoreRepo interface {
- Reader(shard uint, name string, start, end uint64) kv.Reader
- TimeSeriesReader(shard uint, name string, start, end uint64) kv.TimeSeriesReader
- Index(shard uint, name string) kv.IndexStore
-}
-
-// WritePoint is a reference to a underlying area.
-// A single write will be split into several kv writes which can be placed into the same area though an WritePoint
-// even though the area rotation(a new area replaces the old area) takes place.
-// When a new area is created, the old one can be closed until all WritePoint refers to it are closed.
-type WritePoint interface {
- io.Closer
- Writer(shard uint, name string) kv.Writer
- TimeSeriesWriter(shard uint, name string) kv.TimeSeriesWriter
-}
-
-// GetWritePoint returns the WritePoint which refers to the current active area.
-type GetWritePoint func(ts uint64) WritePoint
diff --git a/banyand/stream/service.go b/banyand/stream/service.go
index 9cb37a6..2822fce 100644
--- a/banyand/stream/service.go
+++ b/banyand/stream/service.go
@@ -19,13 +19,20 @@ package stream
import (
"context"
+ "time"
"github.com/pkg/errors"
+ "google.golang.org/protobuf/types/known/timestamppb"
+ "github.com/apache/skywalking-banyandb/api/data"
+ "github.com/apache/skywalking-banyandb/api/event"
commonv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
+ databasev2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v2"
+ "github.com/apache/skywalking-banyandb/banyand/discovery"
"github.com/apache/skywalking-banyandb/banyand/metadata"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
"github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/run"
)
@@ -50,6 +57,9 @@ type service struct {
l *logger.Logger
metadata metadata.Repo
root string
+ pipeline queue.Queue
+ repo discovery.ServiceRepo
+ stopCh chan struct{}
}
func (s *service) Stream(stream *commonv2.Metadata) (Stream, error) {
@@ -107,16 +117,78 @@ func (s *service) PreRun() error {
}
func (s *service) Serve() error {
- panic("implement me")
+ t := time.Now()
+ now := time.Now().UnixNano()
+ nowBp := timestamppb.New(t)
+ for _, sMeta := range s.schemaMap {
+ locator := make([]*databasev2.EntityEvent_TagLocator, 0, len(sMeta.entityLocator))
+ for _, tagLocator := range sMeta.entityLocator {
+ locator = append(locator, &databasev2.EntityEvent_TagLocator{
+ FamilyOffset: uint32(tagLocator.FamilyOffset),
+ TagOffset: uint32(tagLocator.TagOffset),
+ })
+ }
+ _, err := s.repo.Publish(event.TopicEntityEvent, bus.NewMessage(bus.MessageID(now), &databasev2.EntityEvent{
+ Subject: &commonv2.Metadata{
+ Name: sMeta.name,
+ Group: sMeta.group,
+ },
+ EntityLocator: locator,
+ Time: nowBp,
+ Action: databasev2.Action_ACTION_PUT,
+ }))
+ if err != nil {
+ return err
+ }
+ for i := 0; i < int(sMeta.schema.GetShardNum()); i++ {
+ _, errShard := s.repo.Publish(event.TopicShardEvent, bus.NewMessage(bus.MessageID(now), &databasev2.ShardEvent{
+ Shard: &databasev2.Shard{
+ Id: uint64(i),
+ Total: sMeta.schema.GetShardNum(),
+ Metadata: &commonv2.Metadata{
+ Name: sMeta.name,
+ Group: sMeta.group,
+ },
+ Node: &databasev2.Node{
+ Id: s.repo.NodeID(),
+ CreatedAt: nowBp,
+ UpdatedAt: nowBp,
+ Addr: "localhost",
+ },
+ UpdatedAt: nowBp,
+ CreatedAt: nowBp,
+ },
+ Time: nowBp,
+ Action: databasev2.Action_ACTION_PUT,
+ }))
+ if errShard != nil {
+ return errShard
+ }
+ }
+ }
+ errWrite := s.pipeline.Subscribe(data.TopicStreamWrite, s.writeListener)
+ if errWrite != nil {
+ return errWrite
+ }
+ s.stopCh = make(chan struct{})
+ <-s.stopCh
+ return nil
}
func (s *service) GracefulStop() {
- panic("implement me")
+ for _, sm := range s.schemaMap {
+ _ = sm.Close()
+ }
+ if s.stopCh != nil {
+ close(s.stopCh)
+ }
}
//NewService returns a new service
-func NewService(_ context.Context, metadata metadata.Repo, pipeline queue.Queue) (Service, error) {
+func NewService(_ context.Context, metadata metadata.Repo, repo discovery.ServiceRepo, pipeline queue.Queue) (Service, error) {
return &service{
metadata: metadata,
+ repo: repo,
+ pipeline: pipeline,
}, nil
}
diff --git a/banyand/stream/stream.go b/banyand/stream/stream.go
index ffb4fc0..4e47953 100644
--- a/banyand/stream/stream.go
+++ b/banyand/stream/stream.go
@@ -24,18 +24,12 @@ import (
modelv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ "github.com/apache/skywalking-banyandb/pkg/partition"
)
-const strDelimiter = "\n"
-
-type tagIndex struct {
- family int
- tag int
-}
-
type indexRule struct {
rule *databasev2.IndexRule
- tagIndices []tagIndex
+ tagIndices []partition.TagLocator
}
type stream struct {
@@ -44,7 +38,7 @@ type stream struct {
l *logger.Logger
schema *databasev2.Stream
db tsdb.Database
- entityIndex []tagIndex
+ entityLocator partition.EntityLocator
indexRules []*databasev2.IndexRule
indexRuleIndex []indexRule
@@ -63,15 +57,15 @@ func (s *stream) parseSchema() {
for _, tagInEntity := range sm.Entity.GetTagNames() {
fIndex, tIndex, tag := s.findTagByName(tagInEntity)
if tag != nil {
- s.entityIndex = append(s.entityIndex, tagIndex{family: fIndex, tag: tIndex})
+ s.entityLocator = append(s.entityLocator, partition.TagLocator{FamilyOffset: fIndex, TagOffset: tIndex})
}
}
for _, rule := range s.indexRules {
- tagIndices := make([]tagIndex, 0, len(rule.GetTags()))
+ tagIndices := make([]partition.TagLocator, 0, len(rule.GetTags()))
for _, tagInIndex := range rule.GetTags() {
fIndex, tIndex, tag := s.findTagByName(tagInIndex)
if tag != nil {
- tagIndices = append(tagIndices, tagIndex{family: fIndex, tag: tIndex})
+ tagIndices = append(tagIndices, partition.TagLocator{FamilyOffset: fIndex, TagOffset: tIndex})
}
}
s.indexRuleIndex = append(s.indexRuleIndex, indexRule{rule: rule, tagIndices: tagIndices})
diff --git a/banyand/stream/stream_query.go b/banyand/stream/stream_query.go
index ea58fd1..bad6ca3 100644
--- a/banyand/stream/stream_query.go
+++ b/banyand/stream/stream_query.go
@@ -80,7 +80,7 @@ func (s *stream) ParseTagFamily(family string, item tsdb.Item) (*modelv2.TagFami
if err != nil {
return nil, err
}
- tagFamily := &streamv2.ElementValue_TagFamily{}
+ tagFamily := &modelv2.TagFamilyForWrite{}
err = proto.Unmarshal(familyRawBytes, tagFamily)
if err != nil {
return nil, err
diff --git a/banyand/stream/stream_query_test.go b/banyand/stream/stream_query_test.go
index 9f3f305..03eac47 100644
--- a/banyand/stream/stream_query_test.go
+++ b/banyand/stream/stream_query_test.go
@@ -55,7 +55,7 @@ type shardsForTest []shardStruct
func Test_Stream_SelectShard(t *testing.T) {
tester := assert.New(t)
- s, deferFunc := setup(tester)
+ s, deferFunc := setup(t)
defer deferFunc()
_ = setupQueryData(t, "multiple_shards.json", s)
tests := []struct {
@@ -95,8 +95,7 @@ func Test_Stream_SelectShard(t *testing.T) {
}
func Test_Stream_Series(t *testing.T) {
- tester := assert.New(t)
- s, deferFunc := setup(tester)
+ s, deferFunc := setup(t)
defer deferFunc()
baseTime := setupQueryData(t, "multiple_shards.json", s)
tests := []struct {
@@ -537,7 +536,7 @@ func Test_Stream_Series(t *testing.T) {
func Test_Stream_Global_Index(t *testing.T) {
tester := assert.New(t)
- s, deferFunc := setup(tester)
+ s, deferFunc := setup(t)
defer deferFunc()
_ = setupQueryData(t, "global_index.json", s)
tests := []struct {
@@ -720,12 +719,12 @@ func setupQueryData(testing *testing.T, dataFile string, stream *stream) (baseTi
for i, template := range templates {
rawSearchTagFamily, errMarshal := json.Marshal(template)
t.NoError(errMarshal)
- searchTagFamily := &streamv2.ElementValue_TagFamily{}
+ searchTagFamily := &modelv2.TagFamilyForWrite{}
t.NoError(jsonpb.UnmarshalString(string(rawSearchTagFamily), searchTagFamily))
e := &streamv2.ElementValue{
ElementId: strconv.Itoa(i),
Timestamp: timestamppb.New(baseTime.Add(500 * time.Millisecond * time.Duration(i))),
- TagFamilies: []*streamv2.ElementValue_TagFamily{
+ TagFamilies: []*modelv2.TagFamilyForWrite{
{
Tags: []*modelv2.TagValue{
{
diff --git a/banyand/stream/stream_write.go b/banyand/stream/stream_write.go
index 42360f8..f9c54eb 100644
--- a/banyand/stream/stream_write.go
+++ b/banyand/stream/stream_write.go
@@ -18,21 +18,16 @@
package stream
import (
- "bytes"
- "strings"
-
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
"github.com/apache/skywalking-banyandb/api/common"
- "github.com/apache/skywalking-banyandb/api/data"
- modelv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/bus"
- "github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/partition"
+ pbv2 "github.com/apache/skywalking-banyandb/pkg/pb/v2"
)
var (
@@ -41,16 +36,12 @@ var (
)
func (s *stream) Write(value *streamv2.ElementValue) error {
- entity, err := s.buildEntity(value)
- if err != nil {
- return err
- }
- shardID, err := partition.ShardID(entity.Marshal(), s.schema.GetShardNum())
+ entity, shardID, err := s.entityLocator.Locate(value.GetTagFamilies(), s.schema.GetShardNum())
if err != nil {
return err
}
waitCh := make(chan struct{})
- err = s.write(common.ShardID(shardID), value, func() {
+ err = s.write(shardID, tsdb.HashEntity(entity), value, func() {
close(waitCh)
})
if err != nil {
@@ -61,7 +52,7 @@ func (s *stream) Write(value *streamv2.ElementValue) error {
return nil
}
-func (s *stream) write(shardID common.ShardID, value *streamv2.ElementValue, cb callbackFn) error {
+func (s *stream) write(shardID common.ShardID, seriesHashKey []byte, value *streamv2.ElementValue, cb callbackFn) error {
sm := s.schema
fLen := len(value.GetTagFamilies())
if fLen < 1 {
@@ -74,11 +65,7 @@ func (s *stream) write(shardID common.ShardID, value *streamv2.ElementValue, cb
if err != nil {
return err
}
- entity, err := s.buildEntity(value)
- if err != nil {
- return err
- }
- series, err := shard.Series().Get(entity)
+ series, err := shard.Series().GetByHashKey(seriesHashKey)
if err != nil {
return err
}
@@ -154,14 +141,14 @@ func getIndexValue(ruleIndex indexRule, value *streamv2.ElementValue) (val []byt
val = make([]byte, 0, len(ruleIndex.tagIndices))
var existInt bool
for _, tIndex := range ruleIndex.tagIndices {
- tag, err := getTagByOffset(value, tIndex.family, tIndex.tag)
+ tag, err := partition.GetTagByOffset(value.GetTagFamilies(), tIndex.FamilyOffset, tIndex.TagOffset)
if err != nil {
return nil, false, errors.WithMessagef(err, "index rule:%v", ruleIndex.rule.Metadata)
}
if tag.GetInt() != nil {
existInt = true
}
- v, err := marshalIndexFieldValue(tag)
+ v, err := pbv2.MarshalIndexFieldValue(tag)
if err != nil {
return nil, false, err
}
@@ -173,53 +160,6 @@ func getIndexValue(ruleIndex indexRule, value *streamv2.ElementValue) (val []byt
return val, false, nil
}
-func marshalIndexFieldValue(tagValue *modelv2.TagValue) ([]byte, error) {
- switch x := tagValue.GetValue().(type) {
- case *modelv2.TagValue_Str:
- return []byte(x.Str.GetValue()), nil
- case *modelv2.TagValue_Int:
- return convert.Int64ToBytes(x.Int.GetValue()), nil
- case *modelv2.TagValue_StrArray:
- return []byte(strings.Join(x.StrArray.GetValue(), strDelimiter)), nil
- case *modelv2.TagValue_IntArray:
- buf := bytes.NewBuffer(nil)
- for _, i := range x.IntArray.GetValue() {
- buf.Write(convert.Int64ToBytes(i))
- }
- return buf.Bytes(), nil
- case *modelv2.TagValue_BinaryData:
- return x.BinaryData, nil
- }
- return nil, ErrUnsupportedTagForIndexField
-}
-
-func (s *stream) buildEntity(value *streamv2.ElementValue) (tsdb.Entity, error) {
- entity := make(tsdb.Entity, len(s.entityIndex))
- for i, index := range s.entityIndex {
- tag, err := getTagByOffset(value, index.family, index.tag)
- if err != nil {
- return nil, err
- }
- e, errMarshal := marshalIndexFieldValue(tag)
- if errMarshal != nil {
- return nil, errMarshal
- }
- entity[i] = e
- }
- return entity, nil
-}
-
-func getTagByOffset(value *streamv2.ElementValue, fIndex, tIndex int) (*modelv2.TagValue, error) {
- if fIndex >= len(value.TagFamilies) {
- return nil, errors.Wrap(ErrMalformedElement, "tag family offset is invalid")
- }
- family := value.GetTagFamilies()[fIndex]
- if tIndex >= len(family.GetTags()) {
- return nil, errors.Wrap(ErrMalformedElement, "tag offset is invalid")
- }
- return family.GetTags()[tIndex], nil
-}
-
type writeCallback struct {
l *logger.Logger
schemaMap map[string]*stream
@@ -234,14 +174,14 @@ func setUpWriteCallback(l *logger.Logger, schemaMap map[string]*stream) *writeCa
}
func (w *writeCallback) Rev(message bus.Message) (resp bus.Message) {
- writeEvent, ok := message.Data().(data.StreamWriteData)
+ writeEvent, ok := message.Data().(*streamv2.InternalWriteRequest)
if !ok {
w.l.Warn().Msg("invalid event data type")
return
}
- sm := writeEvent.WriteRequest.GetMetadata()
+ sm := writeEvent.GetRequest().GetMetadata()
id := formatStreamID(sm.GetName(), sm.GetGroup())
- err := w.schemaMap[id].write(common.ShardID(writeEvent.ShardID), writeEvent.WriteRequest.GetElement(), nil)
+ err := w.schemaMap[id].write(common.ShardID(writeEvent.GetShardId()), writeEvent.GetSeriesHash(), writeEvent.GetRequest().GetElement(), nil)
if err != nil {
w.l.Debug().Err(err)
}
diff --git a/banyand/stream/stream_write_test.go b/banyand/stream/stream_write_test.go
index 74ae262..6fe079c 100644
--- a/banyand/stream/stream_write_test.go
+++ b/banyand/stream/stream_write_test.go
@@ -20,13 +20,12 @@ package stream
import (
"context"
"encoding/base64"
- "math"
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/apache/skywalking-banyandb/api/common"
commonv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
modelv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
@@ -38,12 +37,11 @@ import (
func Test_Stream_Write(t *testing.T) {
tester := assert.New(t)
- s, deferFunc := setup(tester)
+ s, deferFunc := setup(t)
defer deferFunc()
type args struct {
- shardID uint
- ele *streamv2.ElementValue
+ ele *streamv2.ElementValue
}
tests := []struct {
name string
@@ -53,7 +51,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "golden path",
args: args{
- shardID: 0,
ele: getEle(
"trace_id-xxfff.111323",
0,
@@ -68,7 +65,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "minimal",
args: args{
- shardID: 1,
ele: getEle(
nil,
1,
@@ -80,7 +76,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "http",
args: args{
- shardID: 0,
ele: getEle(
"trace_id-xxfff.111323",
0,
@@ -97,7 +92,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "database",
args: args{
- shardID: 0,
ele: getEle(
"trace_id-xxfff.111323",
0,
@@ -116,7 +110,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "mq",
args: args{
- shardID: 0,
ele: getEle(
"trace_id-xxfff.111323",
1,
@@ -138,7 +131,6 @@ func Test_Stream_Write(t *testing.T) {
{
name: "invalid trace id",
args: args{
- shardID: 1,
ele: getEle(
1212323,
1,
@@ -154,25 +146,8 @@ func Test_Stream_Write(t *testing.T) {
wantErr: true,
},
{
- name: "invalid shard id",
- args: args{
- shardID: math.MaxUint64,
- ele: getEle(
- "trace_id-xxfff.111323",
- 0,
- "webapp_id",
- "10.0.0.1_id",
- "/home_id",
- 300,
- 1622933202000000000,
- ),
- },
- wantErr: true,
- },
- {
name: "unknown tags",
args: args{
- shardID: 0,
ele: getEle(
"trace_id-xxfff.111323",
1,
@@ -197,7 +172,7 @@ func Test_Stream_Write(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- err := s.write(common.ShardID(tt.args.shardID), tt.args.ele, nil)
+ err := s.Write(tt.args.ele)
if tt.wantErr {
tester.Error(err)
return
@@ -208,29 +183,30 @@ func Test_Stream_Write(t *testing.T) {
}
-func setup(t *assert.Assertions) (*stream, func()) {
- t.NoError(logger.Init(logger.Logging{
+func setup(t *testing.T) (*stream, func()) {
+ req := require.New(t)
+ req.NoError(logger.Init(logger.Logging{
Env: "dev",
Level: "info",
}))
- tempDir, deferFunc := test.Space(t)
+ tempDir, deferFunc := test.Space(req)
streamRepo, err := schema.NewStream()
- t.NoError(err)
+ req.NoError(err)
sa, err := streamRepo.Get(context.TODO(), &commonv2.Metadata{
Name: "sw",
Group: "default",
})
- t.NoError(err)
+ req.NoError(err)
mService, err := metadata.NewService(context.TODO())
- t.NoError(err)
+ req.NoError(err)
iRules, err := mService.IndexRules(context.TODO(), sa.Metadata)
- t.NoError(err)
+ req.NoError(err)
sSpec := streamSpec{
schema: sa,
indexRules: iRules,
}
s, err := openStream(tempDir, sSpec, logger.GetLogger("test"))
- t.NoError(err)
+ req.NoError(err)
return s, func() {
_ = s.Close()
deferFunc()
@@ -246,7 +222,7 @@ func getEle(tags ...interface{}) *streamv2.ElementValue {
e := &streamv2.ElementValue{
ElementId: "1231.dfd.123123ssf",
Timestamp: timestamppb.Now(),
- TagFamilies: []*streamv2.ElementValue_TagFamily{
+ TagFamilies: []*modelv2.TagFamilyForWrite{
{
Tags: []*modelv2.TagValue{
{
diff --git a/banyand/tsdb/seriesdb.go b/banyand/tsdb/seriesdb.go
index 422029f..c2e9d5e 100644
--- a/banyand/tsdb/seriesdb.go
+++ b/banyand/tsdb/seriesdb.go
@@ -89,6 +89,7 @@ type SeriesDatabase interface {
io.Closer
GetByID(id common.SeriesID) (Series, error)
Get(entity Entity) (Series, error)
+ GetByHashKey(key []byte) (Series, error)
List(path Path) (SeriesList, error)
}
@@ -110,20 +111,7 @@ type seriesDB struct {
sID common.ShardID
}
-func (s *seriesDB) GetByID(id common.SeriesID) (Series, error) {
- return newSeries(s.context(), id, s), nil
-}
-
-func (s *seriesDB) block(id GlobalItemID) blockDelegate {
- return s.lst[id.segID].lst[id.blockID].delegate()
-}
-
-func (s *seriesDB) shardID() common.ShardID {
- return s.sID
-}
-
-func (s *seriesDB) Get(entity Entity) (Series, error) {
- key := hashEntity(entity)
+func (s *seriesDB) GetByHashKey(key []byte) (Series, error) {
seriesID, err := s.seriesMetadata.Get(key)
if err != nil && err != kv.ErrKeyNotFound {
return nil, err
@@ -141,6 +129,23 @@ func (s *seriesDB) Get(entity Entity) (Series, error) {
return newSeries(s.context(), bytesConvSeriesID(seriesID), s), nil
}
+func (s *seriesDB) GetByID(id common.SeriesID) (Series, error) {
+ return newSeries(s.context(), id, s), nil
+}
+
+func (s *seriesDB) block(id GlobalItemID) blockDelegate {
+ return s.lst[id.segID].lst[id.blockID].delegate()
+}
+
+func (s *seriesDB) shardID() common.ShardID {
+ return s.sID
+}
+
+func (s *seriesDB) Get(entity Entity) (Series, error) {
+ key := HashEntity(entity)
+ return s.GetByHashKey(key)
+}
+
func (s *seriesDB) List(path Path) (SeriesList, error) {
if path.isFull {
id, err := s.seriesMetadata.Get(path.prefix)
@@ -226,7 +231,7 @@ func newSeriesDataBase(ctx context.Context, shardID common.ShardID, path string,
return sdb, nil
}
-func hashEntity(entity Entity) []byte {
+func HashEntity(entity Entity) []byte {
result := make(Entry, 0, len(entity)*8)
for _, entry := range entity {
result = append(result, hash(entry)...)
diff --git a/banyand/tsdb/seriesdb_test.go b/banyand/tsdb/seriesdb_test.go
index 8429295..e5d9a2d 100644
--- a/banyand/tsdb/seriesdb_test.go
+++ b/banyand/tsdb/seriesdb_test.go
@@ -24,6 +24,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/pkg/convert"
@@ -142,7 +143,6 @@ func TestNewPath(t *testing.T) {
}
func Test_SeriesDatabase_Get(t *testing.T) {
-
tests := []struct {
name string
entities []Entity
@@ -174,11 +174,11 @@ func Test_SeriesDatabase_Get(t *testing.T) {
tester := assert.New(t)
tester.NoError(logger.Init(logger.Logging{
Env: "dev",
- Level: "debug",
+ Level: "warn",
}))
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- dir, deferFunc := test.Space(tester)
+ dir, deferFunc := test.Space(require.New(t))
defer deferFunc()
s, err := newSeriesDataBase(context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test")), 0, dir, nil)
tester.NoError(err)
@@ -195,9 +195,9 @@ func Test_SeriesDatabase_List(t *testing.T) {
tester := assert.New(t)
tester.NoError(logger.Init(logger.Logging{
Env: "dev",
- Level: "debug",
+ Level: "warn",
}))
- dir, deferFunc := test.Space(tester)
+ dir, deferFunc := test.Space(require.New(t))
defer deferFunc()
s, err := newSeriesDataBase(context.WithValue(context.Background(), logger.ContextKey, logger.GetLogger("test")), 0, dir, nil)
tester.NoError(err)
@@ -329,7 +329,7 @@ func setUpEntities(t *assert.Assertions, db SeriesDatabase) []*entityWithID {
},
}
for _, d := range data {
- d.id = common.SeriesID(convert.BytesToUint64(hash(hashEntity(d.entity))))
+ d.id = common.SeriesID(convert.BytesToUint64(hash(HashEntity(d.entity))))
series, err := db.Get(d.entity)
t.NoError(err)
t.Greater(uint(series.ID()), uint(0))
diff --git a/banyand/tsdb/tsdb_test.go b/banyand/tsdb/tsdb_test.go
index 1c17c2b..3f91e40 100644
--- a/banyand/tsdb/tsdb_test.go
+++ b/banyand/tsdb/tsdb_test.go
@@ -25,6 +25,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -32,7 +33,7 @@ import (
func TestOpenDatabase(t *testing.T) {
tester := assert.New(t)
- tempDir, deferFunc, _ := setUp(tester)
+ tempDir, deferFunc, _ := setUp(require.New(t))
defer deferFunc()
shardPath := fmt.Sprintf(shardTemplate, tempDir, 0)
validateDirectory(tester, shardPath)
@@ -44,10 +45,10 @@ func TestOpenDatabase(t *testing.T) {
validateDirectory(tester, fmt.Sprintf(blockTemplate, segPath, now.Format(blockFormat)))
}
-func setUp(t *assert.Assertions) (tempDir string, deferFunc func(), db Database) {
+func setUp(t *require.Assertions) (tempDir string, deferFunc func(), db Database) {
t.NoError(logger.Init(logger.Logging{
Env: "dev",
- Level: "debug",
+ Level: "warn",
}))
tempDir, deferFunc = test.Space(t)
db, err := OpenDatabase(
diff --git a/go.mod b/go.mod
index 55e7aa9..aa12e1a 100644
--- a/go.mod
+++ b/go.mod
@@ -7,10 +7,8 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/dgraph-io/badger/v3 v3.2011.1
github.com/dgraph-io/ristretto v0.1.0
- github.com/golang/mock v1.5.0
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6
- github.com/google/uuid v1.3.0
github.com/klauspost/compress v1.13.1 // indirect
github.com/oklog/run v1.1.0
github.com/pkg/errors v0.9.1
@@ -19,7 +17,6 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.7.0
- go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.7.0
golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
diff --git a/go.sum b/go.sum
index 14301da..f8c743c 100644
--- a/go.sum
+++ b/go.sum
@@ -117,7 +117,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -173,8 +172,6 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
diff --git a/pkg/index/inverted/inverted_test.go b/pkg/index/inverted/inverted_test.go
index f08e69f..e56d444 100644
--- a/pkg/index/inverted/inverted_test.go
+++ b/pkg/index/inverted/inverted_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/pkg/index/posting"
"github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
@@ -31,7 +32,7 @@ import (
func TestStore_MatchTerm(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -47,7 +48,7 @@ func TestStore_MatchTerm(t *testing.T) {
func TestStore_MatchTerm_AfterFlush(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -64,7 +65,7 @@ func TestStore_MatchTerm_AfterFlush(t *testing.T) {
func TestStore_Iterator(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -80,7 +81,7 @@ func TestStore_Iterator(t *testing.T) {
func TestStore_Iterator_AfterFlush(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -97,7 +98,7 @@ func TestStore_Iterator_AfterFlush(t *testing.T) {
func TestStore_Iterator_Hybrid(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -132,7 +133,7 @@ func TestStore_Iterator_Hybrid(t *testing.T) {
testcases.RunDuration(t, data, s)
}
-func setUp(t *assert.Assertions) (tempDir string, deferFunc func()) {
+func setUp(t *require.Assertions) (tempDir string, deferFunc func()) {
t.NoError(logger.Init(logger.Logging{
Env: "dev",
Level: "debug",
diff --git a/pkg/index/lsm/lsm_test.go b/pkg/index/lsm/lsm_test.go
index 374d8ef..67b775e 100644
--- a/pkg/index/lsm/lsm_test.go
+++ b/pkg/index/lsm/lsm_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/apache/skywalking-banyandb/pkg/index/testcases"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -29,7 +30,7 @@ import (
func TestStore_MatchTerm(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -45,7 +46,7 @@ func TestStore_MatchTerm(t *testing.T) {
func TestStore_Iterator(t *testing.T) {
tester := assert.New(t)
- path, fn := setUp(tester)
+ path, fn := setUp(require.New(t))
s, err := NewStore(StoreOpts{
Path: path,
Logger: logger.GetLogger("test"),
@@ -59,7 +60,7 @@ func TestStore_Iterator(t *testing.T) {
testcases.RunDuration(t, data, s)
}
-func setUp(t *assert.Assertions) (tempDir string, deferFunc func()) {
+func setUp(t *require.Assertions) (tempDir string, deferFunc func()) {
t.NoError(logger.Init(logger.Logging{
Env: "dev",
Level: "info",
diff --git a/pkg/partition/entity.go b/pkg/partition/entity.go
new file mode 100644
index 0000000..9f73a1b
--- /dev/null
+++ b/pkg/partition/entity.go
@@ -0,0 +1,77 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package partition
+
+import (
+ "github.com/pkg/errors"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ modelv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
+ "github.com/apache/skywalking-banyandb/banyand/tsdb"
+ pbv2 "github.com/apache/skywalking-banyandb/pkg/pb/v2"
+)
+
+var (
+ ErrMalformedElement = errors.New("element is malformed")
+)
+
+type EntityLocator []TagLocator
+
+type TagLocator struct {
+ FamilyOffset int
+ TagOffset int
+}
+
+func (e EntityLocator) Find(value []*modelv2.TagFamilyForWrite) (tsdb.Entity, error) {
+ entity := make(tsdb.Entity, len(e))
+ for i, index := range e {
+ tag, err := GetTagByOffset(value, index.FamilyOffset, index.TagOffset)
+ if err != nil {
+ return nil, err
+ }
+ entry, errMarshal := pbv2.MarshalIndexFieldValue(tag)
+ if errMarshal != nil {
+ return nil, errMarshal
+ }
+ entity[i] = entry
+ }
+ return entity, nil
+}
+
+func (e EntityLocator) Locate(value []*modelv2.TagFamilyForWrite, shardNum uint32) (tsdb.Entity, common.ShardID, error) {
+ entity, err := e.Find(value)
+ if err != nil {
+ return nil, 0, err
+ }
+ id, err := ShardID(entity.Marshal(), shardNum)
+ if err != nil {
+ return nil, 0, err
+ }
+ return entity, common.ShardID(id), nil
+}
+
+func GetTagByOffset(value []*modelv2.TagFamilyForWrite, fIndex, tIndex int) (*modelv2.TagValue, error) {
+ if fIndex >= len(value) {
+ return nil, errors.Wrap(ErrMalformedElement, "tag family offset is invalid")
+ }
+ family := value[fIndex]
+ if tIndex >= len(family.GetTags()) {
+ return nil, errors.Wrap(ErrMalformedElement, "tag offset is invalid")
+ }
+ return family.GetTags()[tIndex], nil
+}
diff --git a/pkg/pb/v2/database.go b/pkg/pb/v2/database.go
deleted file mode 100644
index f0ccecd..0000000
--- a/pkg/pb/v2/database.go
+++ /dev/null
@@ -1,166 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package v2
-
-import (
- "time"
-
- "google.golang.org/protobuf/types/known/timestamppb"
-
- commonv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- databasev2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
-)
-
-type ShardEventBuilder struct {
- se *databasev2.ShardEvent
-}
-
-func NewShardEventBuilder() *ShardEventBuilder {
- return &ShardEventBuilder{se: &databasev2.ShardEvent{}}
-}
-
-func (seb *ShardEventBuilder) Action(action databasev2.Action) *ShardEventBuilder {
- seb.se.Action = action
- return seb
-}
-
-func (seb *ShardEventBuilder) Time(t time.Time) *ShardEventBuilder {
- seb.se.Time = timestamppb.New(t)
- return seb
-}
-
-func (seb *ShardEventBuilder) Shard(shard *databasev2.Shard) *ShardEventBuilder {
- seb.se.Shard = shard
- return seb
-}
-
-func (seb *ShardEventBuilder) Build() *databasev2.ShardEvent {
- return seb.se
-}
-
-type ShardBuilder struct {
- s *databasev2.Shard
-}
-
-func NewShardBuilder() *ShardBuilder {
- return &ShardBuilder{s: &databasev2.Shard{}}
-}
-
-func (sb *ShardBuilder) ID(shardID uint64) *ShardBuilder {
- sb.s.Id = shardID
- return sb
-}
-
-func (sb *ShardBuilder) SeriesMetadata(group, name string) *ShardBuilder {
- sb.s.Series = &commonv2.Metadata{
- Group: group,
- Name: name,
- }
- return sb
-}
-
-func (sb *ShardBuilder) Node(node *databasev2.Node) *ShardBuilder {
- sb.s.Node = node
- return sb
-}
-
-func (sb *ShardBuilder) Total(total uint32) *ShardBuilder {
- sb.s.Total = total
- return sb
-}
-
-func (sb *ShardBuilder) CreatedAt(t time.Time) *ShardBuilder {
- sb.s.CreatedAt = timestamppb.New(t)
- return sb
-}
-
-func (sb *ShardBuilder) UpdatedAt(t time.Time) *ShardBuilder {
- sb.s.UpdatedAt = timestamppb.New(t)
- return sb
-}
-
-func (sb *ShardBuilder) Build() *databasev2.Shard {
- return sb.s
-}
-
-type NodeBuilder struct {
- n *databasev2.Node
-}
-
-func NewNodeBuilder() *NodeBuilder {
- return &NodeBuilder{n: &databasev2.Node{}}
-}
-
-func (nb *NodeBuilder) ID(id string) *NodeBuilder {
- nb.n.Id = id
- return nb
-}
-
-func (nb *NodeBuilder) Addr(addr string) *NodeBuilder {
- nb.n.Addr = addr
- return nb
-}
-
-func (nb *NodeBuilder) UpdatedAt(t time.Time) *NodeBuilder {
- nb.n.UpdatedAt = timestamppb.New(t)
- return nb
-}
-
-func (nb *NodeBuilder) CreatedAt(t time.Time) *NodeBuilder {
- nb.n.CreatedAt = timestamppb.New(t)
- return nb
-}
-
-func (nb *NodeBuilder) Build() *databasev2.Node {
- return nb.n
-}
-
-type SeriesEventBuilder struct {
- se *databasev2.SeriesEvent
-}
-
-func NewSeriesEventBuilder() *SeriesEventBuilder {
- return &SeriesEventBuilder{se: &databasev2.SeriesEvent{}}
-}
-
-func (seb *SeriesEventBuilder) SeriesMetadata(group, name string) *SeriesEventBuilder {
- seb.se.Series = &commonv2.Metadata{
- Group: group,
- Name: name,
- }
- return seb
-}
-
-func (seb *SeriesEventBuilder) FieldNames(names ...string) *SeriesEventBuilder {
- seb.se.FieldNamesCompositeSeriesId = names
- return seb
-}
-
-func (seb *SeriesEventBuilder) Action(action databasev2.Action) *SeriesEventBuilder {
- seb.se.Action = action
- return seb
-}
-
-func (seb *SeriesEventBuilder) Time(t time.Time) *SeriesEventBuilder {
- seb.se.Time = timestamppb.New(t)
- return seb
-}
-
-func (seb *SeriesEventBuilder) Build() *databasev2.SeriesEvent {
- return seb.se
-}
diff --git a/pkg/pb/v2/fields.go b/pkg/pb/v2/fields.go
deleted file mode 100644
index 36714d2..0000000
--- a/pkg/pb/v2/fields.go
+++ /dev/null
@@ -1,151 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package v2
-
-import (
- databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
- "github.com/apache/skywalking-banyandb/pkg/convert"
-)
-
-func Transform(entityValue *tracev1.EntityValue, fieldIndexes []FieldEntry) []*modelv1.TypedPair {
- typedPairs := make([]*modelv1.TypedPair, 0)
- if fieldIndexes == nil {
- return typedPairs
- }
- // copy selected fields
- for _, fieldIndex := range fieldIndexes {
- key, idx, t := fieldIndex.Key, fieldIndex.Index, fieldIndex.Type
- if idx >= len(entityValue.GetFields()) {
- typedPairs = append(typedPairs, &modelv1.TypedPair{
- Key: key,
- Typed: &modelv1.TypedPair_NullPair{
- NullPair: &modelv1.TypedPair_NullWithType{Type: t},
- },
- })
- continue
- }
- f := entityValue.GetFields()[idx]
- switch v := f.GetValueType().(type) {
- case *modelv1.Field_Str:
- typedPairs = append(typedPairs, buildPair(key, v.Str.GetValue()))
- case *modelv1.Field_StrArray:
- typedPairs = append(typedPairs, buildPair(key, v.StrArray.GetValue()))
- case *modelv1.Field_Int:
- typedPairs = append(typedPairs, buildPair(key, v.Int.GetValue()))
- case *modelv1.Field_IntArray:
- typedPairs = append(typedPairs, buildPair(key, v.IntArray.GetValue()))
- case *modelv1.Field_Null:
- typedPairs = append(typedPairs, &modelv1.TypedPair{
- Key: key,
- Typed: &modelv1.TypedPair_NullPair{
- NullPair: &modelv1.TypedPair_NullWithType{Type: t},
- },
- })
- }
- }
- return typedPairs
-}
-
-func buildPair(key string, value interface{}) *modelv1.TypedPair {
- result := &modelv1.TypedPair{
- Key: key,
- }
- switch v := value.(type) {
- case int:
- result.Typed = &modelv1.TypedPair_IntPair{
- IntPair: &modelv1.Int{
- Value: int64(v),
- },
- }
- case []int:
- result.Typed = &modelv1.TypedPair_IntArrayPair{
- IntArrayPair: &modelv1.IntArray{
- Value: convert.IntToInt64(v...),
- },
- }
- case int8:
- result.Typed = &modelv1.TypedPair_IntPair{
- IntPair: &modelv1.Int{
- Value: int64(v),
- },
- }
- case []int8:
- result.Typed = &modelv1.TypedPair_IntArrayPair{
- IntArrayPair: &modelv1.IntArray{
- Value: convert.Int8ToInt64(v...),
- },
- }
- case int16:
- result.Typed = &modelv1.TypedPair_IntPair{
- IntPair: &modelv1.Int{
- Value: int64(v),
- },
- }
- case []int16:
- result.Typed = &modelv1.TypedPair_IntArrayPair{
- IntArrayPair: &modelv1.IntArray{
- Value: convert.Int16ToInt64(v...),
- },
- }
- case int32:
- result.Typed = &modelv1.TypedPair_IntPair{
- IntPair: &modelv1.Int{
- Value: int64(v),
- },
- }
- case []int32:
- result.Typed = &modelv1.TypedPair_IntArrayPair{
- IntArrayPair: &modelv1.IntArray{
- Value: convert.Int32ToInt64(v...),
- },
- }
- case int64:
- result.Typed = &modelv1.TypedPair_IntPair{
- IntPair: &modelv1.Int{
- Value: v,
- },
- }
- case []int64:
- result.Typed = &modelv1.TypedPair_IntArrayPair{
- IntArrayPair: &modelv1.IntArray{
- Value: v,
- },
- }
- case string:
- result.Typed = &modelv1.TypedPair_StrPair{
- StrPair: &modelv1.Str{
- Value: v,
- },
- }
- case []string:
- result.Typed = &modelv1.TypedPair_StrArrayPair{
- StrArrayPair: &modelv1.StrArray{
- Value: v,
- },
- }
- }
- return result
-}
-
-type FieldEntry struct {
- Key string
- Index int
- Type databasev1.FieldType
-}
diff --git a/pkg/pb/v2/write.go b/pkg/pb/v2/write.go
index 31fe826..011daa3 100644
--- a/pkg/pb/v2/write.go
+++ b/pkg/pb/v2/write.go
@@ -18,130 +18,117 @@
package v2
import (
+ "bytes"
+ "strings"
"time"
+ "github.com/pkg/errors"
"google.golang.org/protobuf/types/known/timestamppb"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
+ commonv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v2"
+ modelv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v2"
+ streamv2 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v2"
"github.com/apache/skywalking-banyandb/pkg/convert"
)
-type WriteRequestBuilder struct {
- we *tracev1.WriteRequest
-}
-
-func NewWriteEntityBuilder() *WriteRequestBuilder {
- return &WriteRequestBuilder{we: &tracev1.WriteRequest{}}
-}
-
-func (web *WriteRequestBuilder) Metadata(group, name string) *WriteRequestBuilder {
- web.we.Metadata = &commonv1.Metadata{
- Group: group,
- Name: name,
+const strDelimiter = "\n"
+
+var ErrUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
+
+func MarshalIndexFieldValue(tagValue *modelv2.TagValue) ([]byte, error) {
+ switch x := tagValue.GetValue().(type) {
+ case *modelv2.TagValue_Str:
+ return []byte(x.Str.GetValue()), nil
+ case *modelv2.TagValue_Int:
+ return convert.Int64ToBytes(x.Int.GetValue()), nil
+ case *modelv2.TagValue_StrArray:
+ return []byte(strings.Join(x.StrArray.GetValue(), strDelimiter)), nil
+ case *modelv2.TagValue_IntArray:
+ buf := bytes.NewBuffer(nil)
+ for _, i := range x.IntArray.GetValue() {
+ buf.Write(convert.Int64ToBytes(i))
+ }
+ return buf.Bytes(), nil
+ case *modelv2.TagValue_BinaryData:
+ return x.BinaryData, nil
}
- return web
-}
-
-func (web *WriteRequestBuilder) EntityValue(ev *tracev1.EntityValue) *WriteRequestBuilder {
- web.we.Entity = ev
- return web
+ return nil, ErrUnsupportedTagForIndexField
}
-func (web *WriteRequestBuilder) Build() *tracev1.WriteRequest {
- return web.we
+type StreamWriteRequestBuilder struct {
+ ec *streamv2.WriteRequest
}
-type EntityValueBuilder struct {
- ev *tracev1.EntityValue
+func NewStreamWriteRequestBuilder() *StreamWriteRequestBuilder {
+ return &StreamWriteRequestBuilder{
+ ec: &streamv2.WriteRequest{
+ Element: &streamv2.ElementValue{
+ TagFamilies: make([]*modelv2.TagFamilyForWrite, 0),
+ },
+ },
+ }
}
-func NewEntityValueBuilder() *EntityValueBuilder {
- return &EntityValueBuilder{ev: &tracev1.EntityValue{}}
+func (b *StreamWriteRequestBuilder) Metadata(group, name string) *StreamWriteRequestBuilder {
+ b.ec.Metadata = &commonv2.Metadata{
+ Group: group,
+ Name: name,
+ }
+ return b
}
-func (evb *EntityValueBuilder) EntityID(entityID string) *EntityValueBuilder {
- evb.ev.EntityId = entityID
- return evb
+func (b *StreamWriteRequestBuilder) ID(id string) *StreamWriteRequestBuilder {
+ b.ec.Element.ElementId = id
+ return b
}
-func (evb *EntityValueBuilder) Timestamp(time time.Time) *EntityValueBuilder {
- evb.ev.Timestamp = timestamppb.New(time)
- return evb
+func (b *StreamWriteRequestBuilder) Timestamp(t time.Time) *StreamWriteRequestBuilder {
+ b.ec.Element.Timestamp = timestamppb.New(t)
+ return b
}
-func (evb *EntityValueBuilder) DataBinary(data []byte) *EntityValueBuilder {
- evb.ev.DataBinary = data
- return evb
+func (b *StreamWriteRequestBuilder) TagFamily(tags ...interface{}) *StreamWriteRequestBuilder {
+ tagFamily := &modelv2.TagFamilyForWrite{}
+ for _, tag := range tags {
+ tagFamily.Tags = append(tagFamily.Tags, getTag(tag))
+ }
+ b.ec.Element.TagFamilies = append(b.ec.Element.TagFamilies, tagFamily)
+ return b
}
-func (evb *EntityValueBuilder) Fields(items ...interface{}) *EntityValueBuilder {
- evb.ev.Fields = make([]*modelv1.Field, len(items))
- for idx, item := range items {
- evb.ev.Fields[idx] = buildField(item)
- }
- return evb
+func (b *StreamWriteRequestBuilder) Build() *streamv2.WriteRequest {
+ return b.ec
}
-func buildField(value interface{}) *modelv1.Field {
- if value == nil {
- return &modelv1.Field{ValueType: &modelv1.Field_Null{}}
- }
- switch v := value.(type) {
- case string:
- return &modelv1.Field{
- ValueType: &modelv1.Field_Str{
- Str: &modelv1.Str{
- Value: v,
- },
- },
- }
- case []string:
- return &modelv1.Field{
- ValueType: &modelv1.Field_StrArray{
- StrArray: &modelv1.StrArray{
- Value: v,
- },
- },
+func getTag(tag interface{}) *modelv2.TagValue {
+ if tag == nil {
+ return &modelv2.TagValue{
+ Value: &modelv2.TagValue_Null{},
}
+ }
+ switch t := tag.(type) {
case int:
- return &modelv1.Field{
- ValueType: &modelv1.Field_Int{
- Int: &modelv1.Int{
- Value: int64(v),
+ return &modelv2.TagValue{
+ Value: &modelv2.TagValue_Int{
+ Int: &modelv2.Int{
+ Value: int64(t),
},
},
}
- case []int:
- return &modelv1.Field{
- ValueType: &modelv1.Field_IntArray{
- IntArray: &modelv1.IntArray{
- Value: convert.IntToInt64(v...),
+ case string:
+ return &modelv2.TagValue{
+ Value: &modelv2.TagValue_Str{
+ Str: &modelv2.Str{
+ Value: t,
},
},
}
- case int64:
- return &modelv1.Field{
- ValueType: &modelv1.Field_Int{
- Int: &modelv1.Int{
- Value: v,
- },
+ case []byte:
+ return &modelv2.TagValue{
+ Value: &modelv2.TagValue_BinaryData{
+ BinaryData: t,
},
}
- case []int64:
- return &modelv1.Field{
- ValueType: &modelv1.Field_IntArray{
- IntArray: &modelv1.IntArray{
- Value: v,
- },
- },
- }
- default:
- panic("type not supported")
}
-}
-
-func (evb *EntityValueBuilder) Build() *tracev1.EntityValue {
- return evb.ev
+ return nil
}
diff --git a/pkg/posting/posting.go b/pkg/posting/posting.go
deleted file mode 100644
index d997ab0..0000000
--- a/pkg/posting/posting.go
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package posting
-
-import (
- "github.com/pkg/errors"
-
- "github.com/apache/skywalking-banyandb/api/common"
-)
-
-var ErrListEmpty = errors.New("postings list is empty")
-
-// List is a collection of common.ChunkID.
-type List interface {
- Contains(id common.ChunkID) bool
-
- IsEmpty() bool
-
- Max() (common.ChunkID, error)
-
- Len() int
-
- Iterator() Iterator
-
- Clone() List
-
- Equal(other List) bool
-
- Insert(i common.ChunkID)
-
- Intersect(other List) error
-
- Difference(other List) error
-
- Union(other List) error
-
- UnionMany(others []List) error
-
- AddIterator(iter Iterator) error
-
- AddRange(min, max common.ChunkID) error
-
- RemoveRange(min, max common.ChunkID) error
-
- Reset()
-
- ToSlice() []common.ChunkID
-
- Marshall() ([]byte, error)
-
- Unmarshall(data []byte) error
-}
-
-type Iterator interface {
- Next() bool
-
- Current() common.ChunkID
-
- Close() error
-}
diff --git a/pkg/posting/roaring/roaring.go b/pkg/posting/roaring/roaring.go
deleted file mode 100644
index 63eb2bc..0000000
--- a/pkg/posting/roaring/roaring.go
+++ /dev/null
@@ -1,214 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
... 2909 lines suppressed ...