You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2022/12/05 01:08:12 UTC
[skywalking-banyandb] branch lint created (now e8d655a)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch lint
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
at e8d655a Fix issues found by revive
This branch includes the following new commits:
new e626551 Remove desperated linters
new c3adea8 Fix lint issues in banyand
new e8d655a Fix issues found by revive
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking-banyandb] 01/03: Remove desperated linters
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch lint
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit e626551146266fd864a149c04b2e6a109d27c034
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Dec 1 12:11:25 2022 +0000
Remove desperated linters
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
.github/workflows/flaky-test.yml | 1 +
.github/workflows/slow-test.yml | 1 +
.golangci.yml | 63 +++++++++++++++--
ui/embed.go => api/common/doc.go | 11 +--
api/common/id.go | 1 +
api/common/kind_version.go | 2 +-
ui/embed.go => api/data/doc.go | 11 +--
ui/embed.go => api/event/doc.go | 11 +--
banyand/Makefile | 3 +-
banyand/internal/cmd/root.go | 2 +-
banyand/kv/badger.go | 19 +++---
banyand/kv/kv.go | 30 ++++----
banyand/liaison/grpc/discovery.go | 2 +
banyand/liaison/grpc/measure.go | 6 +-
banyand/liaison/grpc/property.go | 2 +-
banyand/liaison/grpc/registry.go | 12 ++--
banyand/liaison/grpc/server.go | 36 +++++-----
banyand/liaison/grpc/stream.go | 6 +-
banyand/liaison/http/health.go | 2 +-
banyand/liaison/http/server.go | 37 +++++-----
banyand/measure/measure.go | 17 +++--
banyand/measure/measure_suite_test.go | 2 -
banyand/measure/measure_topn.go | 56 +++++++--------
banyand/measure/metadata.go | 4 +-
banyand/measure/service.go | 12 ++--
banyand/metadata/metadata.go | 9 +--
banyand/metadata/metadata_test.go | 6 +-
banyand/metadata/schema/checker.go | 18 +++++
banyand/metadata/schema/etcd.go | 8 ++-
banyand/metadata/schema/etcd_test.go | 8 +--
banyand/metadata/schema/schema.go | 7 +-
banyand/metadata/schema/schema_suite_test.go | 1 -
banyand/observability/metric.go | 29 +++++---
banyand/observability/pprof.go | 35 +++++++---
banyand/observability/type.go | 4 ++
banyand/query/processor_topn.go | 14 ++--
banyand/queue/local.go | 5 +-
banyand/stream/metadata.go | 6 +-
banyand/stream/service.go | 14 ++--
banyand/stream/stream.go | 22 +++---
banyand/stream/stream_suite_test.go | 2 -
banyand/tsdb/block.go | 51 +++++++-------
banyand/tsdb/block_ctrl.go | 18 ++---
banyand/tsdb/bucket/bucket.go | 6 +-
banyand/tsdb/bucket/queue.go | 11 ++-
banyand/tsdb/bucket/strategy.go | 7 +-
banyand/tsdb/bucket/strategy_test.go | 23 +++----
banyand/tsdb/index/writer.go | 44 ++++++------
banyand/tsdb/indexdb.go | 18 ++---
banyand/tsdb/retention.go | 5 +-
banyand/tsdb/scope.go | 4 +-
banyand/tsdb/segment.go | 13 ++--
banyand/tsdb/segment_ctrl.go | 26 +++----
banyand/tsdb/series.go | 18 ++---
banyand/tsdb/series_seek.go | 13 ++--
banyand/tsdb/series_seek_sort.go | 17 +++--
banyand/tsdb/series_write.go | 8 +--
banyand/tsdb/seriesdb.go | 24 +++----
banyand/tsdb/seriesdb_test.go | 4 +-
banyand/tsdb/shard.go | 15 ++--
banyand/tsdb/tsdb.go | 14 ++--
bydbctl/cmd/bydbctl/main.go | 2 +-
bydbctl/internal/cmd/group.go | 12 ++--
bydbctl/internal/cmd/group_test.go | 6 +-
bydbctl/internal/cmd/index_rule.go | 10 +--
bydbctl/internal/cmd/index_rule_binding.go | 10 +--
bydbctl/internal/cmd/index_rule_binding_test.go | 8 +--
bydbctl/internal/cmd/index_rule_test.go | 8 +--
bydbctl/internal/cmd/measure.go | 10 +--
bydbctl/internal/cmd/measure_test.go | 14 ++--
bydbctl/internal/cmd/property.go | 7 +-
bydbctl/internal/cmd/property_test.go | 16 ++---
bydbctl/internal/cmd/root.go | 5 +-
bydbctl/internal/cmd/stream.go | 10 +--
bydbctl/internal/cmd/stream_test.go | 14 ++--
bydbctl/pkg/file/read.go | 2 +-
deprecated-golangci.yml | 57 ----------------
docs/api-reference.md | 4 +-
pkg/Makefile | 2 +-
pkg/bit/reader.go | 12 ++--
pkg/bit/writer.go | 21 +++---
pkg/buffer/writer.go | 2 +-
pkg/bus/bus.go | 24 +++----
pkg/bus/bus_test.go | 9 +--
pkg/config/config.go | 7 +-
pkg/convert/number.go | 4 +-
pkg/encoding/encoding.go | 6 +-
pkg/encoding/int.go | 17 +++--
pkg/encoding/plain.go | 20 +++---
pkg/encoding/xor.go | 25 +++----
pkg/flow/dedup_priority_queue.go | 4 +-
pkg/flow/op.go | 8 +--
pkg/flow/streaming/flow.go | 2 +-
pkg/flow/streaming/flow_sliding_window.go | 39 ++++-------
pkg/flow/streaming/flow_sliding_window_test.go | 18 ++---
pkg/flow/streaming/flow_test.go | 2 +-
pkg/flow/streaming/flow_topn.go | 18 ++---
pkg/flow/streaming/flow_topn_test.go | 2 +-
pkg/flow/streaming/flow_unary.go | 12 ++--
pkg/flow/streaming/sink/slice.go | 6 +-
pkg/flow/streaming/sources/channel.go | 2 +-
pkg/flow/types.go | 4 +-
pkg/grpchelper/client.go | 3 +-
pkg/index/index.go | 4 +-
pkg/index/inverted/inverted.go | 12 ++--
pkg/index/inverted/inverted_test.go | 2 +-
pkg/index/iterator.go | 22 +++---
pkg/index/lsm/lsm.go | 2 +-
pkg/index/testcases/duration.go | 6 +-
pkg/index/testcases/service_name.go | 2 +-
pkg/logger/jsonproto.go | 6 +-
pkg/logger/logger.go | 6 +-
pkg/logger/setting.go | 10 +--
pkg/pb/v1/metadata.go | 60 ++++++++--------
pkg/pb/v1/tsdb/interval.go | 8 +--
pkg/query/logical/common.go | 4 +-
pkg/query/logical/expr.go | 8 +--
pkg/query/logical/index_filter.go | 79 +++++++++++-----------
pkg/query/logical/iter.go | 13 +---
pkg/query/logical/measure/measure_analyzer.go | 4 +-
.../logical/measure/measure_plan_aggregation.go | 6 +-
pkg/query/logical/measure/measure_plan_groupby.go | 13 ++--
.../measure/measure_plan_indexscan_local.go | 24 +++----
pkg/query/logical/measure/measure_top.go | 2 +-
pkg/query/logical/measure/measure_top_test.go | 2 +-
pkg/query/logical/plan_orderby.go | 12 ++--
pkg/query/logical/schema.go | 10 +--
pkg/query/logical/stream/schema.go | 4 +-
pkg/query/logical/stream/stream_analyzer.go | 11 ++-
.../logical/stream/stream_plan_indexscan_local.go | 8 +--
pkg/query/logical/stream/stream_plan_tag_filter.go | 8 ++-
pkg/query/logical/tag_filter.go | 74 ++++++++++----------
pkg/run/closer.go | 7 +-
pkg/run/run.go | 27 ++++----
pkg/run/test.go | 6 +-
pkg/schema/metadata.go | 22 +++---
pkg/test/flow/slice.go | 2 +-
pkg/test/helpers/context.go | 14 ++--
pkg/test/measure/traffic/traffic.go | 5 +-
pkg/test/setup/setup.go | 2 +-
pkg/test/stream/traffic/traffic.go | 9 +--
pkg/timestamp/clock.go | 2 +-
pkg/timestamp/nano.go | 16 ++---
pkg/timestamp/scheduler.go | 17 +++--
pkg/version/version.go | 6 +-
revive.toml | 5 ++
scripts/build/lint-api.mk | 7 +-
scripts/build/lint-bin.mk | 21 ++++++
scripts/build/lint-deprecated.mk | 31 ---------
scripts/build/lint.mk | 4 +-
test/cases/measure/data/data.go | 6 +-
test/cases/measure/measure.go | 4 +-
test/cases/stream/data/data.go | 38 +++++------
test/cases/stream/stream.go | 4 +-
test/cases/topn/data/data.go | 4 +-
test/cases/topn/topn.go | 4 +-
test/e2e-v2/cases/profiling/ebpf/oncpu/sqrt.go | 2 +-
test/integration/other/property_test.go | 68 +++++++++----------
ui/embed.go | 2 +-
159 files changed, 1032 insertions(+), 1065 deletions(-)
diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml
index c076932..0ca35b3 100644
--- a/.github/workflows/flaky-test.yml
+++ b/.github/workflows/flaky-test.yml
@@ -25,6 +25,7 @@ env:
jobs:
test:
+ if: github.repository == 'apache/skywalking-banyandb'
uses: ./.github/workflows/test.yml
with:
options: --repeat 49 --label-filter \\(integration\\&\\&\\!slow\\)\\|\\|banyand
diff --git a/.github/workflows/slow-test.yml b/.github/workflows/slow-test.yml
index f181bd7..cc4fb09 100644
--- a/.github/workflows/slow-test.yml
+++ b/.github/workflows/slow-test.yml
@@ -26,6 +26,7 @@ env:
jobs:
test:
+ if: github.repository == 'apache/skywalking-banyandb'
uses: ./.github/workflows/test.yml
with:
options: --label-filter slow
diff --git a/.golangci.yml b/.golangci.yml
index b6b547f..5094850 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -16,14 +16,31 @@
linters:
disable-all: true
enable:
- - deadcode
+ - asasalint
+ - asciicheck
+ # - containedctx
+ - contextcheck
+ - decorder
+ - dogsled
+ - errname
- errcheck
- - gocritic
+ - errchkjson
+ - errorlint
+ - exhaustive
+ # TODO:// enable this lint
+ # - exhaustruct
+ - exportloopref
+ - gci
- goconst
+ - gocritic
+ - gocyclo
+ - godot
- gofumpt
- - gci
+ - goprintffuncname
+ - gosec
- gosimple
- govet
+ - importas
- ineffassign
- lll
- misspell
@@ -31,10 +48,26 @@ linters:
- stylecheck
- typecheck
- unconvert
- - varcheck
- - prealloc
+ - unparam
- unused
+ - usestdlibvars
+ - whitespace
linters-settings:
+ importas:
+ no-unaliased: true
+ alias:
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1
+ alias: commonv1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1
+ alias: databasev1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1
+ alias: modelv1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1
+ alias: propertyv1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1
+ alias: measurev1
+ - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1
+ alias: streamv1
lll:
line-length: 170
goconst:
@@ -46,6 +79,7 @@ linters-settings:
misspell:
locale: US
unused:
+ go: "1.19"
check-exported: false
unparam:
check-exported: false
@@ -100,6 +134,25 @@ linters-settings:
- unslice
- valSwap
- weakCond
+ godot:
+ # declarations - for top level declaration comments (default);
+ # toplevel - for top level comments;
+ # all - for all comments.
+ scope: toplevel
+ staticcheck:
+ go: "1.19"
+ stylecheck:
+ go: "1.19"
+ exhaustive:
+ check:
+ - switch
+ - map
+ default-signifies-exhaustive: true
+ ignore-enum-members: ".+UNSPECIFIED$"
+ exhaustruct:
+ exclude:
+ - 'cobra\.Command$'
+
run:
deadline: 20m
skip-files:
diff --git a/ui/embed.go b/api/common/doc.go
similarity index 83%
copy from ui/embed.go
copy to api/common/doc.go
index 7d3657c..0ec53bc 100644
--- a/ui/embed.go
+++ b/api/common/doc.go
@@ -15,12 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-// Package ui provides Web based UI for BanyanDB
-package ui
-
-import "embed"
-
-// DistContent contains ui distributed pages
-//
-//go:embed dist
-var DistContent embed.FS
+// Package common defines reusable objects for APIs.
+package common
diff --git a/api/common/id.go b/api/common/id.go
index e106b54..ff0ed50 100644
--- a/api/common/id.go
+++ b/api/common/id.go
@@ -27,6 +27,7 @@ import (
)
type (
+ // SeriesID identit fdf.
SeriesID uint64
ShardID uint32
ItemID uint64
diff --git a/api/common/kind_version.go b/api/common/kind_version.go
index 3da5374..176cd7a 100644
--- a/api/common/kind_version.go
+++ b/api/common/kind_version.go
@@ -17,7 +17,7 @@
package common
-// KindVersion contains elements which identify the API uniquely
+// KindVersion contains elements which identify the API uniquely.
type KindVersion struct {
Version string
Kind string
diff --git a/ui/embed.go b/api/data/doc.go
similarity index 83%
copy from ui/embed.go
copy to api/data/doc.go
index 7d3657c..2591867 100644
--- a/ui/embed.go
+++ b/api/data/doc.go
@@ -15,12 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-// Package ui provides Web based UI for BanyanDB
-package ui
-
-import "embed"
-
-// DistContent contains ui distributed pages
-//
-//go:embed dist
-var DistContent embed.FS
+// Package data contains data transmission topics.
+package data
diff --git a/ui/embed.go b/api/event/doc.go
similarity index 83%
copy from ui/embed.go
copy to api/event/doc.go
index 7d3657c..8a9656c 100644
--- a/ui/embed.go
+++ b/api/event/doc.go
@@ -15,12 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-// Package ui provides Web based UI for BanyanDB
-package ui
-
-import "embed"
-
-// DistContent contains ui distributed pages
-//
-//go:embed dist
-var DistContent embed.FS
+// Package event contains metadata syncing topics.
+package event
diff --git a/banyand/Makefile b/banyand/Makefile
index 6822eb5..56dce25 100644
--- a/banyand/Makefile
+++ b/banyand/Makefile
@@ -28,7 +28,6 @@ include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
include ../scripts/build/build.mk
include ../scripts/build/test.mk
-# TODO: remove this
-include ../scripts/build/lint-deprecated.mk
+include ../scripts/build/lint.mk
include ../scripts/build/docker.mk
include ../scripts/build/help.mk
diff --git a/banyand/internal/cmd/root.go b/banyand/internal/cmd/root.go
index 0bbfbd5..01cc159 100644
--- a/banyand/internal/cmd/root.go
+++ b/banyand/internal/cmd/root.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package cmd is an internal package defining cli commands for BanyanDB
+// Package cmd is an internal package defining cli commands for BanyanDB.
package cmd
import (
diff --git a/banyand/kv/badger.go b/banyand/kv/badger.go
index 9508a16..584a664 100644
--- a/banyand/kv/badger.go
+++ b/banyand/kv/badger.go
@@ -19,6 +19,7 @@ package kv
import (
"bytes"
+ "errors"
"log"
"math"
"time"
@@ -42,10 +43,10 @@ var (
)
type badgerTSS struct {
- shardID int
- dbOpts badger.Options
- db *badger.DB
+ dbOpts badger.Options
badger.TSet
+ db *badger.DB
+ shardID int
}
func (b *badgerTSS) Context(key []byte, ts uint64, n int) (pre Iterator, next Iterator) {
@@ -88,8 +89,8 @@ func (b *badgerTSS) Close() error {
type mergedIter struct {
delegated Iterator
- valid bool
data []byte
+ valid bool
}
func (i *mergedIter) Next() {
@@ -137,9 +138,9 @@ func (i mergedIter) Value() y.ValueStruct {
}
type badgerDB struct {
- shardID int
dbOpts badger.Options
db *badger.DB
+ shardID int
}
func (b *badgerDB) Stats() observability.Statistics {
@@ -172,7 +173,7 @@ func (b *badgerDB) Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error
err := f(b.shardID, k, func() ([]byte, error) {
return y.Copy(it.Value().Value), nil
})
- if err == ErrStopScan {
+ if errors.Is(err, ErrStopScan) {
break
}
if err != nil {
@@ -185,8 +186,8 @@ func (b *badgerDB) Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error
var _ Iterator = (*iterator)(nil)
type iterator struct {
- reverse bool
delegated y.Iterator
+ reverse bool
}
func (i *iterator) Next() {
@@ -255,7 +256,7 @@ func (b *badgerDB) PutWithVersion(key, val []byte, version uint64) error {
func (b *badgerDB) Get(key []byte) ([]byte, error) {
v, err := b.db.Get(y.KeyWithTs(key, math.MaxInt64))
- if err == badger.ErrKeyNotFound {
+ if errors.Is(err, badger.ErrKeyNotFound) {
return nil, ErrKeyNotFound
}
if err != nil {
@@ -283,7 +284,7 @@ func (b *badgerDB) GetAll(key []byte, applyFn func([]byte) error) error {
return ErrKeyNotFound
}
-// badgerLog delegates the zap log to the badger logger
+// badgerLog delegates the zap log to the badger logger.
type badgerLog struct {
*log.Logger
delegated *logger.Logger
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
index 5dd9e53..f180d55 100644
--- a/banyand/kv/kv.go
+++ b/banyand/kv/kv.go
@@ -47,10 +47,10 @@ type Writer interface {
type ScanFunc func(shardID int, key []byte, getVal func() ([]byte, error)) error
type ScanOpts struct {
+ Prefix []byte
PrefetchSize int
PrefetchValues bool
Reverse bool
- Prefix []byte
}
type Reader interface {
@@ -61,7 +61,7 @@ type Reader interface {
Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error
}
-// Store is a common kv storage with auto-generated key
+// Store is a common kv storage with auto-generated key.
type Store interface {
observability.Observable
io.Closer
@@ -83,7 +83,7 @@ type TimeSeriesReader interface {
Context(key []byte, ts uint64, n int) (pre, next Iterator)
}
-// TimeSeriesStore is time series storage
+// TimeSeriesStore is time series storage.
type TimeSeriesStore interface {
observability.Observable
io.Closer
@@ -93,7 +93,7 @@ type TimeSeriesStore interface {
type TimeSeriesOptions func(TimeSeriesStore)
-// TSSWithLogger sets a external logger into underlying TimeSeriesStore
+// TSSWithLogger sets a external logger into underlying TimeSeriesStore.
func TSSWithLogger(l *logger.Logger) TimeSeriesOptions {
return func(store TimeSeriesStore) {
if btss, ok := store.(*badgerTSS); ok {
@@ -161,7 +161,8 @@ type IndexStore interface {
Close() error
}
-// OpenTimeSeriesStore creates a new TimeSeriesStore
+// OpenTimeSeriesStore creates a new TimeSeriesStore.
+// nolint: contextcheck
func OpenTimeSeriesStore(shardID int, path string, options ...TimeSeriesOptions) (TimeSeriesStore, error) {
btss := new(badgerTSS)
btss.shardID = shardID
@@ -177,7 +178,7 @@ func OpenTimeSeriesStore(shardID int, path string, options ...TimeSeriesOptions)
var err error
btss.db, err = badger.Open(btss.dbOpts)
if err != nil {
- return nil, fmt.Errorf("failed to open time series store: %v", err)
+ return nil, fmt.Errorf("failed to open time series store: %w", err)
}
btss.TSet = *badger.NewTSet(btss.db)
return btss, nil
@@ -185,12 +186,12 @@ func OpenTimeSeriesStore(shardID int, path string, options ...TimeSeriesOptions)
type StoreOptions func(Store)
-// StoreWithLogger sets a external logger into underlying Store
+// StoreWithLogger sets a external logger into underlying Store.
func StoreWithLogger(l *logger.Logger) StoreOptions {
return StoreWithNamedLogger("normal-kv", l)
}
-// StoreWithNamedLogger sets a external logger with a name into underlying Store
+// StoreWithNamedLogger sets a external logger with a name into underlying Store.
func StoreWithNamedLogger(name string, l *logger.Logger) StoreOptions {
return func(store Store) {
if bdb, ok := store.(*badgerDB); ok {
@@ -201,7 +202,7 @@ func StoreWithNamedLogger(name string, l *logger.Logger) StoreOptions {
}
}
-// StoreWithMemTableSize sets MemTable size
+// StoreWithMemTableSize sets MemTable size.
func StoreWithMemTableSize(size int64) StoreOptions {
return func(store Store) {
if size < 1 {
@@ -213,7 +214,8 @@ func StoreWithMemTableSize(size int64) StoreOptions {
}
}
-// OpenStore creates a new Store
+// OpenStore creates a new Store.
+// nolint: contextcheck
func OpenStore(shardID int, path string, options ...StoreOptions) (Store, error) {
bdb := new(badgerDB)
bdb.shardID = shardID
@@ -229,14 +231,14 @@ func OpenStore(shardID int, path string, options ...StoreOptions) (Store, error)
var err error
bdb.db, err = badger.Open(bdb.dbOpts)
if err != nil {
- return nil, fmt.Errorf("failed to open normal store: %v", err)
+ return nil, fmt.Errorf("failed to open normal store: %w", err)
}
return bdb, nil
}
type IndexOptions func(store IndexStore)
-// IndexWithLogger sets a external logger into underlying IndexStore
+// IndexWithLogger sets a external logger into underlying IndexStore.
func IndexWithLogger(l *logger.Logger) IndexOptions {
return func(store IndexStore) {
if bdb, ok := store.(*badgerDB); ok {
@@ -247,7 +249,7 @@ func IndexWithLogger(l *logger.Logger) IndexOptions {
}
}
-// OpenIndexStore creates a new IndexStore
+// OpenIndexStore creates a new IndexStore.
func OpenIndexStore(shardID int, path string, options ...IndexOptions) (IndexStore, error) {
bdb := new(badgerDB)
bdb.shardID = shardID
@@ -263,7 +265,7 @@ func OpenIndexStore(shardID int, path string, options ...IndexOptions) (IndexSto
var err error
bdb.db, err = badger.Open(bdb.dbOpts)
if err != nil {
- return nil, fmt.Errorf("failed to index store: %v", err)
+ return nil, fmt.Errorf("failed to index store: %w", err)
}
return bdb, nil
}
diff --git a/banyand/liaison/grpc/discovery.go b/banyand/liaison/grpc/discovery.go
index 4c20e0e..cbff00e 100644
--- a/banyand/liaison/grpc/discovery.go
+++ b/banyand/liaison/grpc/discovery.go
@@ -159,6 +159,8 @@ func (s *entityRepo) Rev(message bus.Message) (resp bus.Message) {
s.entitiesMap[id] = en
case databasev1.Action_ACTION_DELETE:
delete(s.entitiesMap, id)
+ case databasev1.Action_ACTION_UNSPECIFIED:
+ s.log.Warn().RawJSON("event", logger.Proto(e)).Msg("ignored unspecified event")
}
return
}
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index 404f11f..be38a38 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -37,8 +37,8 @@ import (
)
type measureService struct {
- *discoveryService
measurev1.UnimplementedMeasureServiceServer
+ *discoveryService
}
func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) error {
@@ -51,7 +51,7 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
sampled := ms.log.Sample(&zerolog.BasicSampler{N: 10})
for {
writeRequest, err := measure.Recv()
- if err == io.EOF {
+ if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
@@ -108,7 +108,7 @@ func (ms *measureService) Query(_ context.Context, req *measurev1.QueryRequest)
}
msg, errFeat := feat.Get()
if errFeat != nil {
- if errFeat == io.EOF {
+ if errors.Is(errFeat, io.EOF) {
return emptyMeasureQueryResponse, nil
}
return nil, errFeat
diff --git a/banyand/liaison/grpc/property.go b/banyand/liaison/grpc/property.go
index d262797..51b0eba 100644
--- a/banyand/liaison/grpc/property.go
+++ b/banyand/liaison/grpc/property.go
@@ -25,8 +25,8 @@ import (
)
type propertyServer struct {
- schemaRegistry metadata.Service
propertyv1.UnimplementedPropertyServiceServer
+ schemaRegistry metadata.Service
}
func (ps *propertyServer) Apply(ctx context.Context, req *propertyv1.ApplyRequest) (*propertyv1.ApplyResponse, error) {
diff --git a/banyand/liaison/grpc/registry.go b/banyand/liaison/grpc/registry.go
index 8fe916b..767cb6b 100644
--- a/banyand/liaison/grpc/registry.go
+++ b/banyand/liaison/grpc/registry.go
@@ -31,8 +31,8 @@ import (
)
type streamRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedStreamRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (rs *streamRegistryServer) Create(ctx context.Context,
@@ -122,8 +122,8 @@ func groupExist(ctx context.Context, errResource error, metadata *commonv1.Metad
}
type indexRuleBindingRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedIndexRuleBindingRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (rs *indexRuleBindingRegistryServer) Create(ctx context.Context,
@@ -204,8 +204,8 @@ func (rs *indexRuleBindingRegistryServer) Exist(ctx context.Context, req *databa
}
type indexRuleRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedIndexRuleRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (rs *indexRuleRegistryServer) Create(ctx context.Context, req *databasev1.IndexRuleRegistryServiceCreateRequest) (
@@ -280,8 +280,8 @@ func (rs *indexRuleRegistryServer) Exist(ctx context.Context, req *databasev1.In
}
type measureRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedMeasureRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (rs *measureRegistryServer) Create(ctx context.Context, req *databasev1.MeasureRegistryServiceCreateRequest) (
@@ -354,8 +354,8 @@ func (rs *measureRegistryServer) Exist(ctx context.Context, req *databasev1.Meas
}
type groupRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedGroupRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (rs *groupRegistryServer) Create(ctx context.Context, req *databasev1.GroupRegistryServiceCreateRequest) (
@@ -428,8 +428,8 @@ func (rs *groupRegistryServer) Exist(ctx context.Context, req *databasev1.GroupR
}
type topNAggregationRegistryServer struct {
- schemaRegistry metadata.Service
databasev1.UnimplementedTopNAggregationRegistryServiceServer
+ schemaRegistry metadata.Service
}
func (ts *topNAggregationRegistryServer) Create(ctx context.Context,
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index 66cc2b4..4aa66ba 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -52,28 +52,26 @@ var (
)
type Server struct {
- addr string
- maxRecvMsgSize int
- tls bool
- certFile string
- keyFile string
- log *logger.Logger
- ser *grpclib.Server
- pipeline queue.Queue
- repo discovery.ServiceRepo
- creds credentials.TransportCredentials
-
- stopCh chan struct{}
-
+ pipeline queue.Queue
+ creds credentials.TransportCredentials
+ repo discovery.ServiceRepo
+ stopCh chan struct{}
+ *measureRegistryServer
+ log *logger.Logger
+ ser *grpclib.Server
+ *propertyServer
+ *topNAggregationRegistryServer
+ *groupRegistryServer
+ *indexRuleRegistryServer
streamSVC *streamService
measureSVC *measureService
*streamRegistryServer
*indexRuleBindingRegistryServer
- *indexRuleRegistryServer
- *measureRegistryServer
- *groupRegistryServer
- *topNAggregationRegistryServer
- *propertyServer
+ addr string
+ keyFile string
+ certFile string
+ maxRecvMsgSize int
+ tls bool
}
func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRepo, schemaRegistry metadata.Service) *Server {
@@ -113,9 +111,9 @@ func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRe
func (s *Server) PreRun() error {
s.log = logger.GetLogger("liaison-grpc")
components := []struct {
+ discoverySVC *discoveryService
shardEvent bus.Topic
entityEvent bus.Topic
- discoverySVC *discoveryService
}{
{
shardEvent: event.StreamTopicShardEvent,
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index bf5ef5a..191df78 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -37,8 +37,8 @@ import (
)
type streamService struct {
- *discoveryService
streamv1.UnimplementedStreamServiceServer
+ *discoveryService
}
func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
@@ -51,7 +51,7 @@ func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
sampled := s.log.Sample(&zerolog.BasicSampler{N: 10})
for {
writeEntity, err := stream.Recv()
- if err == io.EOF {
+ if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
@@ -113,7 +113,7 @@ func (s *streamService) Query(_ context.Context, req *streamv1.QueryRequest) (*s
message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), req)
feat, errQuery := s.pipeline.Publish(data.TopicStreamQuery, message)
if errQuery != nil {
- if errQuery == io.EOF {
+ if errors.Is(errQuery, io.EOF) {
return emptyStreamQueryResponse, nil
}
return nil, errQuery
diff --git a/banyand/liaison/http/health.go b/banyand/liaison/http/health.go
index 3912c7b..7681e21 100644
--- a/banyand/liaison/http/health.go
+++ b/banyand/liaison/http/health.go
@@ -58,7 +58,7 @@ type healthCheckClient struct {
func (g *healthCheckClient) Check(ctx context.Context, r *grpc_health_v1.HealthCheckRequest, opts ...grpc.CallOption) (*grpc_health_v1.HealthCheckResponse, error) {
var resp *grpc_health_v1.HealthCheckResponse
- if err := grpchelper.Request(context.Background(), 10*time.Second, func(rpcCtx context.Context) (err error) {
+ if err := grpchelper.Request(ctx, 10*time.Second, func(rpcCtx context.Context) (err error) {
resp, err = grpc_health_v1.NewHealthClient(g.conn).Check(rpcCtx,
&grpc_health_v1.HealthCheckRequest{
Service: "",
diff --git a/banyand/liaison/http/server.go b/banyand/liaison/http/server.go
index 8b90c4d..4f51e25 100644
--- a/banyand/liaison/http/server.go
+++ b/banyand/liaison/http/server.go
@@ -23,6 +23,7 @@ import (
"io/fs"
"net/http"
"strings"
+ "time"
"github.com/go-chi/chi/v5"
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
@@ -30,10 +31,10 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measure_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- property_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
- stream_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/run"
"github.com/apache/skywalking-banyandb/ui"
@@ -53,14 +54,13 @@ func NewService() ServiceRepo {
}
type service struct {
- listenAddr string
- grpcAddr string
mux *chi.Mux
stopCh chan struct{}
clientCloser context.CancelFunc
l *logger.Logger
-
- srv *http.Server
+ srv *http.Server
+ listenAddr string
+ grpcAddr string
}
func (p *service) FlagSet() *run.FlagSet {
@@ -91,8 +91,9 @@ func (p *service) PreRun() error {
serveIndex := serveFileContents("index.html", httpFS)
p.mux.Mount("/", intercept404(fileServer, serveIndex))
p.srv = &http.Server{
- Addr: p.listenAddr,
- Handler: p.mux,
+ Addr: p.listenAddr,
+ Handler: p.mux,
+ ReadHeaderTimeout: 3 * time.Second,
}
return nil
}
@@ -112,14 +113,14 @@ func (p *service) Serve() run.StopNotify {
}
gwMux := runtime.NewServeMux(runtime.WithHealthzEndpoint(client))
err = multierr.Combine(
- database_v1.RegisterStreamRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- database_v1.RegisterMeasureRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- database_v1.RegisterIndexRuleRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- database_v1.RegisterIndexRuleBindingRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- database_v1.RegisterGroupRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- stream_v1.RegisterStreamServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- measure_v1.RegisterMeasureServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
- property_v1.RegisterPropertyServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ databasev1.RegisterStreamRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ databasev1.RegisterMeasureRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ databasev1.RegisterIndexRuleRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ databasev1.RegisterIndexRuleBindingRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ databasev1.RegisterGroupRegistryServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ streamv1.RegisterStreamServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ measurev1.RegisterMeasureServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
+ propertyv1.RegisterPropertyServiceHandlerFromEndpoint(ctx, gwMux, p.grpcAddr, opts),
)
if err != nil {
p.l.Error().Err(err).Msg("Failed to register endpoints")
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 971f08e..0cc528f 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -39,19 +39,18 @@ const (
)
type measure struct {
- name string
- group string
- shardNum uint32
- l *logger.Logger
- schema *databasev1.Measure
- // maxObservedModRevision is the max observed revision of index rules in the spec
- maxObservedModRevision int64
databaseSupplier tsdb.Supplier
+ l *logger.Logger
+ schema *databasev1.Measure
+ indexWriter *index.Writer
+ processorManager *topNProcessorManager
+ name string
+ group string
entityLocator partition.EntityLocator
indexRules []*databasev1.IndexRule
- indexWriter *index.Writer
+ maxObservedModRevision int64
interval time.Duration
- processorManager *topNProcessorManager
+ shardNum uint32
}
func (s *measure) GetSchema() *databasev1.Measure {
diff --git a/banyand/measure/measure_suite_test.go b/banyand/measure/measure_suite_test.go
index 9f6cc38..d1711b9 100644
--- a/banyand/measure/measure_suite_test.go
+++ b/banyand/measure/measure_suite_test.go
@@ -42,7 +42,6 @@ func TestMeasure(t *testing.T) {
ginkgo.RunSpecs(t, "Measure Suite")
}
-// BeforeSuite - Init logger
var _ = ginkgo.BeforeSuite(func() {
gomega.Expect(logger.Init(logger.Logging{
Env: "dev",
@@ -50,7 +49,6 @@ var _ = ginkgo.BeforeSuite(func() {
})).To(gomega.Succeed())
})
-// service to preload measure
type preloadMeasureService struct {
metaSvc metadata.Service
}
diff --git a/banyand/measure/measure_topn.go b/banyand/measure/measure_topn.go
index 2454d72..e5ad23d 100644
--- a/banyand/measure/measure_topn.go
+++ b/banyand/measure/measure_topn.go
@@ -66,18 +66,18 @@ var (
)
type topNStreamingProcessor struct {
- flow.ComponentState
+ databaseSupplier tsdb.Supplier
+ streamingFlow flow.Flow
l *logger.Logger
- shardNum uint32
- interval time.Duration
topNSchema *databasev1.TopNAggregation
- sortDirection modelv1.Sort
- databaseSupplier tsdb.Supplier
src chan interface{}
in chan flow.StreamRecord
errCh <-chan error
stopCh chan struct{}
- streamingFlow flow.Flow
+ flow.ComponentState
+ interval time.Duration
+ shardNum uint32
+ sortDirection modelv1.Sort
}
func (t *topNStreamingProcessor) In() chan<- flow.StreamRecord {
@@ -98,7 +98,7 @@ func (t *topNStreamingProcessor) run(ctx context.Context) {
if !ok {
return
}
- if err := t.writeStreamRecord(record); err != nil {
+ if err := t.writeStreamRecord(ctx, record); err != nil {
t.l.Err(err).Msg("fail to write stream record")
}
case <-ctx.Done():
@@ -124,7 +124,7 @@ func (t *topNStreamingProcessor) Close() error {
return err
}
-func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) error {
+func (t *topNStreamingProcessor) writeStreamRecord(ctx context.Context, record flow.StreamRecord) error {
tuples, ok := record.Data().([]*streaming.Tuple2)
if !ok {
return errors.New("invalid data type")
@@ -141,12 +141,12 @@ func (t *topNStreamingProcessor) writeStreamRecord(record flow.StreamRecord) err
for rankNum, tuple := range tuples {
fieldValue := tuple.V1.(int64)
data := tuple.V2.(flow.StreamRecord).Data().(flow.Data)
- err = multierr.Append(err, t.writeData(eventTime, timeBucket, fieldValue, data, rankNum))
+ err = multierr.Append(err, t.writeData(ctx, eventTime, timeBucket, fieldValue, data, rankNum))
}
return err
}
-func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket string, fieldValue int64, data flow.Data, rankNum int) error {
+func (t *topNStreamingProcessor) writeData(ctx context.Context, eventTime time.Time, timeBucket string, fieldValue int64, data flow.Data, rankNum int) error {
var tagValues []*modelv1.TagValue
if len(t.topNSchema.GetGroupByTagNames()) > 0 {
var ok bool
@@ -166,7 +166,7 @@ func (t *topNStreamingProcessor) writeData(eventTime time.Time, timeBucket strin
if err != nil {
return err
}
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
span, err := series.Create(ctx, eventTime)
if err != nil {
@@ -301,16 +301,13 @@ func (t *topNStreamingProcessor) handleError() {
t.stopCh <- struct{}{}
}
-// topNProcessorManager manages multiple topNStreamingProcessor(s) belonging to a single measure
+// topNProcessorManager manages multiple topNStreamingProcessor(s) belonging to a single measure.
type topNProcessorManager struct {
- // RWMutex here is to protect the processorMap from data race, i.e.
- // the send operation to the underlying channel vs. the close of the channel
- // TODO: this can be optimized if the bus Listener can be synchronously finished,
- sync.RWMutex
l *logger.Logger
m *measure
- topNSchemas []*databasev1.TopNAggregation
processorMap map[*commonv1.Metadata][]*topNStreamingProcessor
+ topNSchemas []*databasev1.TopNAggregation
+ sync.RWMutex
}
func (manager *topNProcessorManager) Close() error {
@@ -417,7 +414,7 @@ func (manager *topNProcessorManager) buildFilterForCriteria(criteria *modelv1.Cr
}
// buildFilterForCondition builds a logical and composable filter for a logical expression which have underlying conditions,
-// or nested logical expressions as its children
+// or nested logical expressions as its children.
func (manager *topNProcessorManager) buildFilterForLogicalExpr(logicalExpr *modelv1.LogicalExpression) (conditionFilter, error) {
left, lErr := manager.buildFilterForCriteria(logicalExpr.Left)
if lErr != nil {
@@ -437,7 +434,7 @@ func composeWithOp(left, right conditionFilter, op modelv1.LogicalExpression_Log
return &orFilter{left, right}
}
-// buildFilterForCondition builds a single, composable filter for a single condition
+// buildFilterForCondition builds a single, composable filter for a single condition.
func (manager *topNProcessorManager) buildFilterForCondition(cond *modelv1.Condition) (conditionFilter, error) {
familyOffset, tagOffset, spec := pbv1.FindTagByName(manager.m.GetSchema().GetTagFamilies(), cond.GetName())
if spec == nil {
@@ -529,9 +526,9 @@ type conditionFilter interface {
}
type strTagFilter struct {
- partition.TagLocator
- op modelv1.Condition_BinaryOp
val string
+ partition.TagLocator
+ op modelv1.Condition_BinaryOp
}
func (f *strTagFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) bool {
@@ -541,14 +538,15 @@ func (f *strTagFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) bool
return strValue == f.val
case modelv1.Condition_BINARY_OP_NE:
return strValue != f.val
+ default:
+ return false
}
- return false
}
type idTagFilter struct {
- partition.TagLocator
- op modelv1.Condition_BinaryOp
val string
+ partition.TagLocator
+ op modelv1.Condition_BinaryOp
}
func (f *idTagFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) bool {
@@ -558,8 +556,9 @@ func (f *idTagFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) bool {
return val == f.val
case modelv1.Condition_BINARY_OP_NE:
return val != f.val
+ default:
+ return false
}
- return false
}
type int64TagFilter struct {
@@ -583,8 +582,9 @@ func (f *int64TagFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) boo
return val <= f.val
case modelv1.Condition_BINARY_OP_LT:
return val < f.val
+ default:
+ return false
}
- return false
}
type andFilter struct {
@@ -603,10 +603,10 @@ func (f *orFilter) predicate(tagFamilies []*modelv1.TagFamilyForWrite) bool {
return f.l.predicate(tagFamilies) || f.r.predicate(tagFamilies)
}
-// groupTagsLocator can be used to locate tags within families
+// groupTagsLocator can be used to locate tags within families.
type groupTagsLocator []partition.TagLocator
-// newGroupLocator generates a groupTagsLocator which strictly preserve the order of groupByNames
+// newGroupLocator generates a groupTagsLocator which strictly preserve the order of groupByNames.
func newGroupLocator(m *databasev1.Measure, groupByNames []string) (groupTagsLocator, error) {
groupTags := make([]partition.TagLocator, 0, len(groupByNames))
for _, groupByName := range groupByNames {
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index 549c838..88aa56c 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -185,10 +185,10 @@ func (sr *schemaRepo) loadMeasure(metadata *commonv1.Metadata) (*measure, bool)
var _ resourceSchema.ResourceSupplier = (*supplier)(nil)
type supplier struct {
- path string
- dbOpts tsdb.DatabaseOpts
metadata metadata.Repo
l *logger.Logger
+ path string
+ dbOpts tsdb.DatabaseOpts
}
func newSupplier(path string, metadata metadata.Repo, dbOpts tsdb.DatabaseOpts, l *logger.Logger) *supplier {
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 9b828cf..6b15ede 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -52,17 +52,15 @@ type Service interface {
var _ Service = (*service)(nil)
type service struct {
- root string
- dbOpts tsdb.DatabaseOpts
-
schemaRepo schemaRepo
writeListener bus.MessageListener
- l *logger.Logger
metadata metadata.Repo
pipeline queue.Queue
repo discovery.ServiceRepo
- // stop channel for the service
- stopCh chan struct{}
+ l *logger.Logger
+ stopCh chan struct{}
+ root string
+ dbOpts tsdb.DatabaseOpts
}
func (s *service) Measure(metadata *commonv1.Metadata) (Measure, error) {
@@ -154,7 +152,7 @@ func (s *service) GracefulStop() {
}
}
-// NewService returns a new service
+// NewService returns a new service.
func NewService(_ context.Context, metadata metadata.Repo, repo discovery.ServiceRepo, pipeline queue.Queue) (Service, error) {
return &service{
metadata: metadata,
diff --git a/banyand/metadata/metadata.go b/banyand/metadata/metadata.go
index 8ecdb31..acaf677 100644
--- a/banyand/metadata/metadata.go
+++ b/banyand/metadata/metadata.go
@@ -31,7 +31,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-// IndexFilter provides methods to find a specific index related objects and vice versa
+// IndexFilter provides methods to find a specific index related objects and vice versa.
type IndexFilter interface {
// IndexRules fetches v1.IndexRule by subject defined in IndexRuleBinding
IndexRules(ctx context.Context, subject *commonv1.Metadata) ([]*databasev1.IndexRule, error)
@@ -171,7 +171,6 @@ func (s *service) IndexRules(ctx context.Context, subject *commonv1.Metadata) ([
continue
}
result = append(result, r)
-
}
return result, indexRuleErr
}
@@ -201,7 +200,7 @@ func (s *service) Subjects(ctx context.Context, indexRule *databasev1.IndexRule,
switch catalog {
case commonv1.Catalog_CATALOG_STREAM:
- stream, getErr := s.schemaRegistry.GetStream(context.TODO(), &commonv1.Metadata{
+ stream, getErr := s.schemaRegistry.GetStream(ctx, &commonv1.Metadata{
Name: sub.GetName(),
Group: indexRule.GetMetadata().GetGroup(),
})
@@ -210,7 +209,7 @@ func (s *service) Subjects(ctx context.Context, indexRule *databasev1.IndexRule,
}
foundSubjects = append(foundSubjects, stream)
case commonv1.Catalog_CATALOG_MEASURE:
- measure, getErr := s.schemaRegistry.GetMeasure(context.TODO(), &commonv1.Metadata{
+ measure, getErr := s.schemaRegistry.GetMeasure(ctx, &commonv1.Metadata{
Name: sub.GetName(),
Group: indexRule.GetMetadata().GetGroup(),
})
@@ -218,6 +217,8 @@ func (s *service) Subjects(ctx context.Context, indexRule *databasev1.IndexRule,
subjectErr = multierr.Append(subjectErr, getErr)
}
foundSubjects = append(foundSubjects, measure)
+ default:
+ continue
}
}
diff --git a/banyand/metadata/metadata_test.go b/banyand/metadata/metadata_test.go
index 682c762..ebdf335 100644
--- a/banyand/metadata/metadata_test.go
+++ b/banyand/metadata/metadata_test.go
@@ -100,16 +100,16 @@ func Test_service_RulesBySubject(t *testing.T) {
t.Errorf("RulesBySubject() error = %v, wantErr %v", err, tt.wantErr)
return
}
- is.Equal(getIndexRule(s, tt.want...), got)
+ is.Equal(getIndexRule(ctx, s, tt.want...), got)
})
}
}
-func getIndexRule(s Service, names ...string) []*databasev1.IndexRule {
+func getIndexRule(ctx context.Context, s Service, names ...string) []*databasev1.IndexRule {
ruleRepo := s.IndexRuleRegistry()
result := make([]*databasev1.IndexRule, 0, len(names))
for _, name := range names {
- indexRule, _ := ruleRepo.GetIndexRule(context.TODO(), &commonv1.Metadata{
+ indexRule, _ := ruleRepo.GetIndexRule(ctx, &commonv1.Metadata{
Group: "default",
Name: name,
})
diff --git a/banyand/metadata/schema/checker.go b/banyand/metadata/schema/checker.go
index 7f0b61e..4a1a875 100644
--- a/banyand/metadata/schema/checker.go
+++ b/banyand/metadata/schema/checker.go
@@ -24,6 +24,7 @@ import (
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
)
type equalityChecker func(a, b proto.Message) bool
@@ -67,4 +68,21 @@ var checkerMap = map[Kind]equalityChecker{
protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
protocmp.Transform())
},
+ KindTopNAggregation: func(a, b proto.Message) bool {
+ return cmp.Equal(a, b,
+ protocmp.IgnoreUnknown(),
+ protocmp.IgnoreFields(&databasev1.TopNAggregation{}, "updated_at"),
+ protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
+ protocmp.Transform())
+ },
+ KindProperty: func(a, b proto.Message) bool {
+ return cmp.Equal(a, b,
+ protocmp.IgnoreUnknown(),
+ protocmp.IgnoreFields(&propertyv1.Property{}, "updated_at"),
+ protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
+ protocmp.Transform())
+ },
+ KindMask: func(a, b proto.Message) bool {
+ return false
+ },
}
diff --git a/banyand/metadata/schema/etcd.go b/banyand/metadata/schema/etcd.go
index 8fc654d..f3b3fa9 100644
--- a/banyand/metadata/schema/etcd.go
+++ b/banyand/metadata/schema/etcd.go
@@ -73,8 +73,8 @@ func ConfigureListener(lc, lp string) RegistryOption {
}
type eventHandler struct {
- interestKeys Kind
handler EventHandler
+ interestKeys Kind
}
func (eh *eventHandler) InterestOf(kind Kind) bool {
@@ -233,7 +233,7 @@ func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) erro
}
modRevision := getResp.Kvs[0].ModRevision
- txnResp, txnErr := e.kv.Txn(context.Background()).
+ txnResp, txnErr := e.kv.Txn(ctx).
If(clientv3.Compare(clientv3.ModRevision(key), "=", modRevision)).
Then(clientv3.OpPut(key, string(val))).
Commit()
@@ -329,6 +329,10 @@ func (e *etcdSchemaRegistry) delete(ctx context.Context, metadata Metadata) (boo
message = &databasev1.IndexRule{}
case KindProperty:
message = &propertyv1.Property{}
+ case KindTopNAggregation:
+ message = &databasev1.TopNAggregation{}
+ default:
+ return false, nil
}
if unmarshalErr := proto.Unmarshal(resp.PrevKvs[0].Value, message); unmarshalErr == nil {
e.notifyDelete(Metadata{
diff --git a/banyand/metadata/schema/etcd_test.go b/banyand/metadata/schema/etcd_test.go
index 795e4a3..5532d24 100644
--- a/banyand/metadata/schema/etcd_test.go
+++ b/banyand/metadata/schema/etcd_test.go
@@ -145,9 +145,9 @@ func Test_Etcd_Entity_Get(t *testing.T) {
tester.NoError(err)
tests := []struct {
- name string
meta *commonv1.Metadata
get func(Registry, *commonv1.Metadata) (HasMetadata, error)
+ name string
expectedErr bool
}{
{
@@ -237,8 +237,8 @@ func Test_Etcd_Entity_List(t *testing.T) {
tester.NoError(err)
tests := []struct {
- name string
list func(Registry) (int, error)
+ name string
expectedLen int
}{
{
@@ -319,9 +319,9 @@ func Test_Etcd_Delete(t *testing.T) {
tester.NoError(err)
tests := []struct {
- name string
list func(Registry) (int, error)
delete func(Registry) error
+ name string
expectedLenBefore int
expectedLenAfter int
}{
@@ -388,9 +388,9 @@ func Test_Notify(t *testing.T) {
req.NoError(err)
tests := []struct {
- name string
testFunc func(context.Context, Registry) error
validationFunc func(*mockedEventHandler) bool
+ name string
}{
{
name: "modify indexRule",
diff --git a/banyand/metadata/schema/schema.go b/banyand/metadata/schema/schema.go
index 62de89c..02aacd3 100644
--- a/banyand/metadata/schema/schema.go
+++ b/banyand/metadata/schema/schema.go
@@ -69,17 +69,14 @@ type Registry interface {
}
type TypeMeta struct {
- Kind Kind
Name string
Group string
+ Kind Kind
}
type Metadata struct {
- TypeMeta
-
- // Spec holds the configuration object as a protobuf message
- // Or a metadataHolder as a container
Spec Spec
+ TypeMeta
}
type Spec interface{}
diff --git a/banyand/metadata/schema/schema_suite_test.go b/banyand/metadata/schema/schema_suite_test.go
index 51afd65..c5f64d3 100644
--- a/banyand/metadata/schema/schema_suite_test.go
+++ b/banyand/metadata/schema/schema_suite_test.go
@@ -32,7 +32,6 @@ func TestSchema(t *testing.T) {
ginkgo.RunSpecs(t, "Schema Suite")
}
-// BeforeSuite - Init logger
var _ = ginkgo.BeforeSuite(func() {
Expect(logger.Init(logger.Logging{
Env: "dev",
diff --git a/banyand/observability/metric.go b/banyand/observability/metric.go
index 180bc84..b4598c6 100644
--- a/banyand/observability/metric.go
+++ b/banyand/observability/metric.go
@@ -18,6 +18,7 @@ package observability
import (
"net/http"
+ "time"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -32,14 +33,15 @@ var (
func NewMetricService() run.Service {
return &metricService{
- stopCh: make(chan struct{}),
+ closer: run.NewCloser(1),
}
}
type metricService struct {
- listenAddr string
- stopCh chan struct{}
l *logger.Logger
+ svr *http.Server
+ closer *run.Closer
+ listenAddr string
}
func (p *metricService) FlagSet() *run.FlagSet {
@@ -49,6 +51,9 @@ func (p *metricService) FlagSet() *run.FlagSet {
}
func (p *metricService) Validate() error {
+ if p.listenAddr == "" {
+ return errNoAddr
+ }
return nil
}
@@ -58,16 +63,22 @@ func (p *metricService) Name() string {
func (p *metricService) Serve() run.StopNotify {
p.l = logger.GetLogger(p.Name())
- http.Handle("/metrics", promhttp.Handler())
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.Handler())
+ p.svr = &http.Server{
+ Addr: p.listenAddr,
+ ReadHeaderTimeout: 3 * time.Second,
+ Handler: mux,
+ }
go func() {
+ defer p.closer.Done()
p.l.Info().Str("listenAddr", p.listenAddr).Msg("Start metric server")
- _ = http.ListenAndServe(p.listenAddr, nil)
- p.stopCh <- struct{}{}
+ _ = p.svr.ListenAndServe()
}()
-
- return p.stopCh
+ return p.closer.CloseNotify()
}
func (p *metricService) GracefulStop() {
- close(p.stopCh)
+ _ = p.svr.Close()
+ p.closer.CloseThenWait()
}
diff --git a/banyand/observability/pprof.go b/banyand/observability/pprof.go
index 9a07a2d..1340f9b 100644
--- a/banyand/observability/pprof.go
+++ b/banyand/observability/pprof.go
@@ -19,8 +19,8 @@ package observability
import (
"net/http"
- // Register pprof package
- _ "net/http/pprof"
+ "net/http/pprof"
+ "time"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/run"
@@ -33,14 +33,15 @@ var (
func NewProfService() run.Service {
return &pprofService{
- stopCh: make(chan struct{}),
+ closer: run.NewCloser(1),
}
}
type pprofService struct {
- listenAddr string
- stopCh chan struct{}
l *logger.Logger
+ svr *http.Server
+ closer *run.Closer
+ listenAddr string
}
func (p *pprofService) FlagSet() *run.FlagSet {
@@ -50,6 +51,9 @@ func (p *pprofService) FlagSet() *run.FlagSet {
}
func (p *pprofService) Validate() error {
+ if p.listenAddr == "" {
+ return errNoAddr
+ }
return nil
}
@@ -59,15 +63,26 @@ func (p *pprofService) Name() string {
func (p *pprofService) Serve() run.StopNotify {
p.l = logger.GetLogger(p.Name())
+ mux := http.NewServeMux()
+ mux.HandleFunc("/debug/pprof/", pprof.Index)
+ mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+ mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+ mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+ mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+ p.svr = &http.Server{
+ Addr: p.listenAddr,
+ ReadHeaderTimeout: 3 * time.Second,
+ Handler: mux,
+ }
go func() {
+ defer p.closer.Done()
p.l.Info().Str("listenAddr", p.listenAddr).Msg("Start pprof server")
- _ = http.ListenAndServe(p.listenAddr, nil)
- p.stopCh <- struct{}{}
+ _ = p.svr.ListenAndServe()
}()
-
- return p.stopCh
+ return p.closer.CloseNotify()
}
func (p *pprofService) GracefulStop() {
- close(p.stopCh)
+ _ = p.svr.Close()
+ p.closer.CloseThenWait()
}
diff --git a/banyand/observability/type.go b/banyand/observability/type.go
index ab8cb1b..f6829d1 100644
--- a/banyand/observability/type.go
+++ b/banyand/observability/type.go
@@ -16,6 +16,10 @@
// under the License.
package observability
+import "errors"
+
+var errNoAddr = errors.New("no address")
+
type Statistics struct {
MemBytes int64
MaxMemBytes int64
diff --git a/banyand/query/processor_topn.go b/banyand/query/processor_topn.go
index bfb3d2c..59abc23 100644
--- a/banyand/query/processor_topn.go
+++ b/banyand/query/processor_topn.go
@@ -236,12 +236,12 @@ func (t *topNQueryProcessor) scanSeries(series tsdb.Series, request *measurev1.T
var _ heap.Interface = (*postAggregationProcessor)(nil)
type aggregatorItem struct {
- key string
int64Func aggregation.Int64Func
+ key string
index int
}
-// postProcessor defines necessary methods for Top-N post processor with or without aggregation
+// postProcessor defines necessary methods for Top-N post processor with or without aggregation.
type postProcessor interface {
put(key string, val int64, timestampMillis uint64) error
val() []*measurev1.TopNList
@@ -266,14 +266,14 @@ func createTopNPostAggregator(topN int32, aggrFunc modelv1.AggregationFunction,
return aggregator
}
-// postAggregationProcessor is an implementation of postProcessor with aggregation
+// postAggregationProcessor is an implementation of postProcessor with aggregation.
type postAggregationProcessor struct {
+ cache map[string]*aggregatorItem
+ items []*aggregatorItem
+ latestTimestamp uint64
topN int32
sort modelv1.Sort
aggrFunc modelv1.AggregationFunction
- items []*aggregatorItem
- cache map[string]*aggregatorItem
- latestTimestamp uint64
}
func (aggr postAggregationProcessor) Len() int {
@@ -393,9 +393,9 @@ func (n *nonAggregatorItem) SetIndex(i int) {
}
type postNonAggregationProcessor struct {
+ timelines map[uint64]*flow.DedupPriorityQueue
topN int32
sort modelv1.Sort
- timelines map[uint64]*flow.DedupPriorityQueue
}
func (naggr *postNonAggregationProcessor) val() []*measurev1.TopNList {
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index 60dd747..1061cbf 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package queue implements the data transmission queue
package queue
import (
@@ -34,7 +35,7 @@ type local struct {
stopCh chan struct{}
}
-// GracefulStop implements Queue
+// GracefulStop implements Queue.
func (l *local) GracefulStop() {
l.local.Close()
if l.stopCh != nil {
@@ -42,7 +43,7 @@ func (l *local) GracefulStop() {
}
}
-// Serve implements Queue
+// Serve implements Queue.
func (l *local) Serve() run.StopNotify {
return l.stopCh
}
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index f4fba35..5b200d9 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -172,10 +172,10 @@ func (sr *schemaRepo) loadStream(metadata *commonv1.Metadata) (*stream, bool) {
var _ resourceSchema.ResourceSupplier = (*supplier)(nil)
type supplier struct {
- path string
- dbOpts tsdb.DatabaseOpts
metadata metadata.Repo
l *logger.Logger
+ path string
+ dbOpts tsdb.DatabaseOpts
}
func newSupplier(path string, metadata metadata.Repo, dbOpts tsdb.DatabaseOpts, l *logger.Logger) *supplier {
@@ -192,7 +192,7 @@ func (s *supplier) OpenResource(shardNum uint32, db tsdb.Supplier, spec resource
return openStream(shardNum, db, streamSpec{
schema: streamSchema,
indexRules: spec.IndexRules,
- }, s.l)
+ }, s.l), nil
}
func (s *supplier) ResourceSchema(repo metadata.Repo, md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
diff --git a/banyand/stream/service.go b/banyand/stream/service.go
index 184e8d3..77c4989 100644
--- a/banyand/stream/service.go
+++ b/banyand/stream/service.go
@@ -50,17 +50,15 @@ type Service interface {
var _ Service = (*service)(nil)
type service struct {
- root string
- dbOpts tsdb.DatabaseOpts
-
schemaRepo schemaRepo
- writeListener *writeCallback
- l *logger.Logger
metadata metadata.Repo
pipeline queue.Queue
repo discovery.ServiceRepo
- // stop channel for the service
- stopCh chan struct{}
+ writeListener *writeCallback
+ l *logger.Logger
+ stopCh chan struct{}
+ root string
+ dbOpts tsdb.DatabaseOpts
}
func (s *service) Stream(metadata *commonv1.Metadata) (Stream, error) {
@@ -147,7 +145,7 @@ func (s *service) GracefulStop() {
}
}
-// NewService returns a new service
+// NewService returns a new service.
func NewService(_ context.Context, metadata metadata.Repo, repo discovery.ServiceRepo, pipeline queue.Queue) (Service, error) {
return &service{
metadata: metadata,
diff --git a/banyand/stream/stream.go b/banyand/stream/stream.go
index 1af3871..41f45a4 100644
--- a/banyand/stream/stream.go
+++ b/banyand/stream/stream.go
@@ -30,24 +30,22 @@ import (
"github.com/apache/skywalking-banyandb/pkg/schema"
)
-// a chunk is 1MB
+// a chunk is 1MB.
const chunkSize = 1 << 20
var _ schema.Resource = (*stream)(nil)
type stream struct {
- name string
- group string
- shardNum uint32
- l *logger.Logger
- // schema is the reference to the spec of the stream
- schema *databasev1.Stream
- // maxObservedModRevision is the max observed revision of index rules in the spec
- maxObservedModRevision int64
db tsdb.Supplier
+ l *logger.Logger
+ schema *databasev1.Stream
+ indexWriter *index.Writer
+ name string
+ group string
entityLocator partition.EntityLocator
indexRules []*databasev1.IndexRule
- indexWriter *index.Writer
+ maxObservedModRevision int64
+ shardNum uint32
}
func (s *stream) GetMetadata() *commonv1.Metadata {
@@ -81,7 +79,7 @@ type streamSpec struct {
indexRules []*databasev1.IndexRule
}
-func openStream(shardNum uint32, db tsdb.Supplier, spec streamSpec, l *logger.Logger) (*stream, error) {
+func openStream(shardNum uint32, db tsdb.Supplier, spec streamSpec, l *logger.Logger) *stream {
sm := &stream{
shardNum: shardNum,
schema: spec.schema,
@@ -99,5 +97,5 @@ func openStream(shardNum uint32, db tsdb.Supplier, spec streamSpec, l *logger.Lo
IndexRules: spec.indexRules,
EnableGlobalIndex: true,
})
- return sm, nil
+ return sm
}
diff --git a/banyand/stream/stream_suite_test.go b/banyand/stream/stream_suite_test.go
index d3177ad..acc9955 100644
--- a/banyand/stream/stream_suite_test.go
+++ b/banyand/stream/stream_suite_test.go
@@ -41,7 +41,6 @@ func TestStream(t *testing.T) {
RunSpecs(t, "Stream Suite")
}
-// BeforeSuite - Init logger
var _ = BeforeSuite(func() {
Expect(logger.Init(logger.Logging{
Env: "dev",
@@ -49,7 +48,6 @@ var _ = BeforeSuite(func() {
})).To(Succeed())
})
-// service to preload stream
type preloadStreamService struct {
metaSvc metadata.Service
}
diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go
index 4ab5fc6..d5c48cf 100644
--- a/banyand/tsdb/block.go
+++ b/banyand/tsdb/block.go
@@ -56,40 +56,39 @@ const (
var ErrBlockClosingInterrupted = errors.New("interrupt to close the block")
type block struct {
- path string
- l *logger.Logger
- queue bucket.Queue
- suffix string
- ref *atomic.Int32
- closed *atomic.Bool
- deleted *atomic.Bool
- lock sync.RWMutex
- position common.Position
- memSize int64
- lsmMemSize int64
-
- store kv.TimeSeriesStore
- invertedIndex index.Store
- lsmIndex index.Store
- closableLst []io.Closer
+ encodingMethod EncodingMethod
+ store kv.TimeSeriesStore
+ queue bucket.Queue
+ bucket.Reporter
clock timestamp.Clock
+ lsmIndex index.Store
+ invertedIndex index.Store
+ closed *atomic.Bool
+ l *logger.Logger
+ deleted *atomic.Bool
+ ref *atomic.Int32
+ position common.Position
timestamp.TimeRange
- bucket.Reporter
- segID SectionID
- blockID SectionID
- segSuffix string
- encodingMethod EncodingMethod
+ segSuffix string
+ suffix string
+ path string
+ closableLst []io.Closer
+ lsmMemSize int64
+ memSize int64
+ lock sync.RWMutex
+ segID SectionID
+ blockID SectionID
}
type blockOpts struct {
- segID SectionID
- segSuffix string
- blockSize IntervalRule
+ queue bucket.Queue
+ scheduler *timestamp.Scheduler
timeRange timestamp.TimeRange
+ segSuffix string
suffix string
path string
- queue bucket.Queue
- scheduler *timestamp.Scheduler
+ blockSize IntervalRule
+ segID SectionID
}
func newBlock(ctx context.Context, opts blockOpts) (b *block, err error) {
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
index 2df4160..475b218 100644
--- a/banyand/tsdb/block_ctrl.go
+++ b/banyand/tsdb/block_ctrl.go
@@ -33,19 +33,18 @@ import (
)
type blockController struct {
- sync.RWMutex
segCtx context.Context
- segID SectionID
+ blockQueue bucket.Queue
+ clock timestamp.Clock
+ scheduler *timestamp.Scheduler
+ l *logger.Logger
+ segTimeRange timestamp.TimeRange
segSuffix string
location string
- segTimeRange timestamp.TimeRange
- blockSize IntervalRule
lst []*block
- blockQueue bucket.Queue
- scheduler *timestamp.Scheduler
- clock timestamp.Clock
-
- l *logger.Logger
+ blockSize IntervalRule
+ sync.RWMutex
+ segID SectionID
}
func newBlockController(segCtx context.Context, segID SectionID, segSuffix, location string, segTimeRange timestamp.TimeRange,
@@ -276,6 +275,7 @@ func (bc *blockController) create(start time.Time) (*block, error) {
return b, nil
}
+// nolint: contextcheck
func (bc *blockController) load(startTime, endTime time.Time, root string) (b *block, err error) {
suffix := bc.Format(startTime)
if b, err = newBlock(
diff --git a/banyand/tsdb/bucket/bucket.go b/banyand/tsdb/bucket/bucket.go
index 297a1dd..6bdb01f 100644
--- a/banyand/tsdb/bucket/bucket.go
+++ b/banyand/tsdb/bucket/bucket.go
@@ -70,11 +70,11 @@ func (*dummyReporter) String() string {
}
type timeBasedReporter struct {
- timestamp.TimeRange
- name string
+ clock timestamp.Clock
scheduler *timestamp.Scheduler
count *atomic.Uint32
- clock timestamp.Clock
+ timestamp.TimeRange
+ name string
}
func NewTimeBasedReporter(name string, timeRange timestamp.TimeRange, clock timestamp.Clock, scheduler *timestamp.Scheduler) Reporter {
diff --git a/banyand/tsdb/bucket/queue.go b/banyand/tsdb/bucket/queue.go
index 5128406..18e900a 100644
--- a/banyand/tsdb/bucket/queue.go
+++ b/banyand/tsdb/bucket/queue.go
@@ -54,15 +54,14 @@ const (
var ErrInvalidSize = errors.New("invalid size")
type lruQueue struct {
- l *logger.Logger
- size int
- recentSize int
- evictSize int
- evictFn EvictFn
-
recent simplelru.LRUCache
frequent simplelru.LRUCache
recentEvict simplelru.LRUCache
+ l *logger.Logger
+ evictFn EvictFn
+ size int
+ recentSize int
+ evictSize int
lock sync.RWMutex
}
diff --git a/banyand/tsdb/bucket/strategy.go b/banyand/tsdb/bucket/strategy.go
index 14fae60..dda86d8 100644
--- a/banyand/tsdb/bucket/strategy.go
+++ b/banyand/tsdb/bucket/strategy.go
@@ -38,13 +38,12 @@ type Ratio float64
type Strategy struct {
optionsErr error
- ratio Ratio
ctrl Controller
current atomic.Value
- currentRatio uint64
logger *logger.Logger
-
- closer *run.Closer
+ closer *run.Closer
+ ratio Ratio
+ currentRatio uint64
}
type StrategyOptions func(*Strategy)
diff --git a/banyand/tsdb/bucket/strategy_test.go b/banyand/tsdb/bucket/strategy_test.go
index 7752cff..b365905 100644
--- a/banyand/tsdb/bucket/strategy_test.go
+++ b/banyand/tsdb/bucket/strategy_test.go
@@ -37,7 +37,7 @@ var _ = Describe("Strategy", func() {
Context("Applying the strategy", func() {
var strategy *bucket.Strategy
It("uses the golden settings", func() {
- ctrl := newController(2, 1, 10)
+ ctrl := newController(2, 1)
var err error
strategy, err = bucket.NewStrategy(ctrl)
Expect(err).NotTo(HaveOccurred())
@@ -45,7 +45,7 @@ var _ = Describe("Strategy", func() {
Eventually(ctrl.isFull, flags.EventuallyTimeout).Should(BeTrue())
})
It("never reaches the limit", func() {
- ctrl := newController(1, 0, 10)
+ ctrl := newController(1, 0)
var err error
strategy, err = bucket.NewStrategy(ctrl)
Expect(err).NotTo(HaveOccurred())
@@ -53,7 +53,7 @@ var _ = Describe("Strategy", func() {
Consistently(ctrl.isFull).ShouldNot(BeTrue())
})
It("exceeds the limit", func() {
- ctrl := newController(2, 3, 10)
+ ctrl := newController(2, 3)
var err error
strategy, err = bucket.NewStrategy(ctrl)
Expect(err).NotTo(HaveOccurred())
@@ -61,7 +61,7 @@ var _ = Describe("Strategy", func() {
Eventually(ctrl.isFull, flags.EventuallyTimeout).Should(BeTrue())
})
It("'s first step exceeds the limit", func() {
- ctrl := newController(2, 15, 10)
+ ctrl := newController(2, 15)
var err error
strategy, err = bucket.NewStrategy(ctrl)
Expect(err).NotTo(HaveOccurred())
@@ -76,7 +76,7 @@ var _ = Describe("Strategy", func() {
})
Context("Invalid parameter", func() {
It("passes a ratio > 1.0", func() {
- ctrl := newController(2, 3, 10)
+ ctrl := newController(2, 3)
_, err := bucket.NewStrategy(ctrl, bucket.WithNextThreshold(1.1))
Expect(err).To(MatchError(bucket.ErrInvalidParameter))
})
@@ -84,17 +84,16 @@ var _ = Describe("Strategy", func() {
})
type controller struct {
+ reporter *reporter
maxBuckets int
usedBuckets int
-
- reporter *reporter
- capacity int
- step int
- mux sync.RWMutex
+ capacity int
+ step int
+ mux sync.RWMutex
}
-func newController(maxBuckets, step, capacity int) *controller {
- ctrl := &controller{step: step, maxBuckets: maxBuckets, capacity: capacity}
+func newController(maxBuckets, step int) *controller {
+ ctrl := &controller{step: step, maxBuckets: maxBuckets, capacity: 10}
ctrl.newReporter()
return ctrl
}
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
index 1424be0..5e98d04 100644
--- a/banyand/tsdb/index/writer.go
+++ b/banyand/tsdb/index/writer.go
@@ -38,22 +38,22 @@ import (
type CallbackFn func()
type Message struct {
- Scope tsdb.Entry
Value Value
LocalWriter tsdb.Writer
BlockCloser io.Closer
+ Scope tsdb.Entry
}
type Value struct {
- TagFamilies []*modelv1.TagFamilyForWrite
Timestamp time.Time
+ TagFamilies []*modelv1.TagFamilyForWrite
}
type WriterOptions struct {
- ShardNum uint32
+ DB tsdb.Supplier
Families []*databasev1.TagFamilySpec
IndexRules []*databasev1.IndexRule
- DB tsdb.Supplier
+ ShardNum uint32
EnableGlobalIndex bool
}
@@ -65,11 +65,11 @@ const (
)
type Writer struct {
- l *logger.Logger
db tsdb.Supplier
+ l *logger.Logger
+ invertRuleIndex map[byte][]*partition.IndexRuleLocator
shardNum uint32
enableGlobalIndex bool
- invertRuleIndex map[byte][]*partition.IndexRuleLocator
}
func NewWriter(ctx context.Context, options WriterOptions) *Writer {
@@ -96,12 +96,18 @@ func NewWriter(ctx context.Context, options WriterOptions) *Writer {
continue
}
key |= global
+ case databasev1.IndexRule_LOCATION_UNSPECIFIED:
+ w.l.Warn().RawJSON("index-rule", logger.Proto(ruleIndex.Rule)).Msg("invalid:unspecified location")
+ continue
}
switch rule.Type {
case databasev1.IndexRule_TYPE_INVERTED:
key |= inverted
case databasev1.IndexRule_TYPE_TREE:
key |= tree
+ case databasev1.IndexRule_TYPE_UNSPECIFIED:
+ w.l.Warn().RawJSON("index-rule", logger.Proto(ruleIndex.Rule)).Msg("invalid:unspecified type")
+ continue
}
rules := w.invertRuleIndex[key]
rules = append(rules, ruleIndex)
@@ -125,12 +131,12 @@ func (s *Writer) Close() error {
return nil
}
-// TODO: should listen to pipeline in a distributed cluster
+// TODO: should listen to pipeline in a distributed cluster.
func (s *Writer) writeGlobalIndex(scope tsdb.Entry, ref tsdb.GlobalItemID, value Value) error {
collect := func(ruleIndexes []*partition.IndexRuleLocator, fn func(indexWriter tsdb.IndexWriter, fields []index.Field) error) error {
fields := make(map[uint][]index.Field)
for _, ruleIndex := range ruleIndexes {
- values, _, err := getIndexValue(ruleIndex, value)
+ values, err := getIndexValue(ruleIndex, value)
if err != nil {
return err
}
@@ -189,7 +195,7 @@ func (s *Writer) writeLocalIndex(writer tsdb.Writer, value Value) (err error) {
collect := func(ruleIndexes []*partition.IndexRuleLocator, fn func(fields []index.Field) error) error {
fields := make([]index.Field, 0)
for _, ruleIndex := range ruleIndexes {
- values, _, err := getIndexValue(ruleIndex, value)
+ values, err := getIndexValue(ruleIndex, value)
if err != nil {
return err
}
@@ -221,11 +227,10 @@ func (s *Writer) writeLocalIndex(writer tsdb.Writer, value Value) (err error) {
var ErrUnsupportedIndexType = errors.New("unsupported index type")
-func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]byte, isInt bool, err error) {
+func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]byte, err error) {
val = make([][]byte, 0)
- var existInt bool
if len(ruleIndex.TagIndices) != 1 {
- return nil, false, errors.WithMessagef(ErrUnsupportedIndexType,
+ return nil, errors.WithMessagef(ErrUnsupportedIndexType,
"the index rule %s(%v) didn't support composited tags",
ruleIndex.Rule.Metadata.Name, ruleIndex.Rule.Tags)
}
@@ -233,29 +238,26 @@ func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]
tag, err := partition.GetTagByOffset(value.TagFamilies, tIndex.FamilyOffset, tIndex.TagOffset)
if errors.Is(err, partition.ErrMalformedElement) {
- return val, false, nil
+ return val, nil
}
if err != nil {
- return nil, false, errors.WithMessagef(err, "index rule:%v", ruleIndex.Rule.Metadata)
- }
- if tag.GetInt() != nil {
- existInt = true
+ return nil, errors.WithMessagef(err, "index rule:%v", ruleIndex.Rule.Metadata)
}
fv, err := pbv1.ParseTagValue(tag)
if errors.Is(err, pbv1.ErrNullValue) {
- return nil, existInt, nil
+ return nil, nil
}
if err != nil {
- return nil, false, err
+ return nil, err
}
v := fv.GetValue()
if v != nil {
val = append(val, v)
- return val, existInt, nil
+ return val, nil
}
arr := fv.GetArr()
if arr != nil {
val = append(val, arr...)
}
- return val, existInt, nil
+ return val, nil
}
diff --git a/banyand/tsdb/indexdb.go b/banyand/tsdb/indexdb.go
index 4add96f..797b111 100644
--- a/banyand/tsdb/indexdb.go
+++ b/banyand/tsdb/indexdb.go
@@ -52,8 +52,8 @@ type IndexSeekBuilder interface{}
var _ IndexDatabase = (*indexDB)(nil)
type indexDB struct {
- shardID common.ShardID
segCtrl *segmentController
+ shardID common.ShardID
}
func (i *indexDB) Seek(field index.Field) ([]GlobalItemID, error) {
@@ -72,7 +72,7 @@ func (i *indexDB) Seek(field index.Field) ([]GlobalItemID, error) {
result = append(result, *id)
return nil
})
- if err == kv.ErrKeyNotFound {
+ if errors.Is(err, kv.ErrKeyNotFound) {
return result, nil
}
}
@@ -83,20 +83,20 @@ func (i *indexDB) WriterBuilder() IndexWriterBuilder {
return newIndexWriterBuilder(i.segCtrl)
}
-func newIndexDatabase(_ context.Context, id common.ShardID, segCtrl *segmentController) (IndexDatabase, error) {
+func newIndexDatabase(_ context.Context, id common.ShardID, segCtrl *segmentController) IndexDatabase {
return &indexDB{
shardID: id,
segCtrl: segCtrl,
- }, nil
+ }
}
var _ IndexWriterBuilder = (*indexWriterBuilder)(nil)
type indexWriterBuilder struct {
- scope Entry
- segCtrl *segmentController
ts time.Time
+ segCtrl *segmentController
globalItemID *GlobalItemID
+ scope Entry
}
func (i *indexWriterBuilder) Scope(scope Entry) IndexWriterBuilder {
@@ -115,7 +115,7 @@ func (i *indexWriterBuilder) GlobalItemID(itemID GlobalItemID) IndexWriterBuilde
}
func (i *indexWriterBuilder) Build() (IndexWriter, error) {
- seg, err := i.segCtrl.create(i.ts, false)
+ seg, err := i.segCtrl.create(i.ts)
if err != nil {
return nil, err
}
@@ -139,10 +139,10 @@ func newIndexWriterBuilder(segCtrl *segmentController) IndexWriterBuilder {
var _ IndexWriter = (*indexWriter)(nil)
type indexWriter struct {
- scope Entry
- seg *segment
ts time.Time
+ seg *segment
itemID *GlobalItemID
+ scope Entry
}
func (i *indexWriter) WriteLSMIndex(fields []index.Field) (err error) {
diff --git a/banyand/tsdb/retention.go b/banyand/tsdb/retention.go
index b25ae20..6eb2e0f 100644
--- a/banyand/tsdb/retention.go
+++ b/banyand/tsdb/retention.go
@@ -27,10 +27,9 @@ import (
)
type retentionTask struct {
- segment *segmentController
-
- option cron.ParseOption
+ segment *segmentController
expr string
+ option cron.ParseOption
duration time.Duration
}
diff --git a/banyand/tsdb/scope.go b/banyand/tsdb/scope.go
index f65764f..ab9711c 100644
--- a/banyand/tsdb/scope.go
+++ b/banyand/tsdb/scope.go
@@ -27,8 +27,8 @@ import (
var _ Shard = (*ScopedShard)(nil)
type ScopedShard struct {
- scope Entry
delegated Shard
+ scope Entry
}
func NewScopedShard(scope Entry, delegated Shard) Shard {
@@ -69,8 +69,8 @@ func (sd *ScopedShard) State() ShardState {
var _ SeriesDatabase = (*scopedSeriesDatabase)(nil)
type scopedSeriesDatabase struct {
- scope Entry
delegated SeriesDatabase
+ scope Entry
}
func (sdd *scopedSeriesDatabase) Stats() observability.Statistics {
diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go
index d4d346e..4a21a71 100644
--- a/banyand/tsdb/segment.go
+++ b/banyand/tsdb/segment.go
@@ -35,17 +35,16 @@ import (
var ErrEndOfSegment = errors.New("reached the end of the segment")
type segment struct {
- id SectionID
- path string
- suffix string
-
globalIndex kv.Store
- l *logger.Logger
- timestamp.TimeRange
bucket.Reporter
+ l *logger.Logger
blockController *blockController
blockManageStrategy *bucket.Strategy
- closeOnce sync.Once
+ timestamp.TimeRange
+ path string
+ suffix string
+ closeOnce sync.Once
+ id SectionID
}
func openSegment(ctx context.Context, startTime, endTime time.Time, path, suffix string,
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
index 4d6cd30..de02ef1 100644
--- a/banyand/tsdb/segment_ctrl.go
+++ b/banyand/tsdb/segment_ctrl.go
@@ -34,17 +34,16 @@ import (
)
type segmentController struct {
- sync.RWMutex
shardCtx context.Context
+ blockQueue bucket.Queue
+ clock timestamp.Clock
+ scheduler *timestamp.Scheduler
+ l *logger.Logger
location string
+ lst []*segment
segmentSize IntervalRule
blockSize IntervalRule
- lst []*segment
- blockQueue bucket.Queue
- scheduler *timestamp.Scheduler
- clock timestamp.Clock
-
- l *logger.Logger
+ sync.RWMutex
}
func newSegmentController(shardCtx context.Context, location string, segmentSize, blockSize IntervalRule,
@@ -126,7 +125,7 @@ func (sc *segmentController) Current() (bucket.Reporter, error) {
}(); b != nil {
return b, nil
}
- return sc.create(now, true)
+ return sc.create(now)
}
func (sc *segmentController) Next() (bucket.Reporter, error) {
@@ -135,7 +134,7 @@ func (sc *segmentController) Next() (bucket.Reporter, error) {
return nil, err
}
seg := c.(*segment)
- reporter, err := sc.create(sc.segmentSize.NextTime(seg.Start), true)
+ reporter, err := sc.create(sc.segmentSize.NextTime(seg.Start))
if errors.Is(err, ErrEndOfSegment) {
return nil, bucket.ErrNoMoreBucket
}
@@ -187,7 +186,7 @@ func (sc *segmentController) open() error {
sc.Lock()
defer sc.Unlock()
return loadSections(sc.location, sc, sc.segmentSize, func(start, end time.Time) error {
- _, err := sc.load(start, end, sc.location, false)
+ _, err := sc.load(start, end, sc.location)
if errors.Is(err, ErrEndOfSegment) {
return nil
}
@@ -195,7 +194,7 @@ func (sc *segmentController) open() error {
})
}
-func (sc *segmentController) create(start time.Time, createBlockIfEmpty bool) (*segment, error) {
+func (sc *segmentController) create(start time.Time) (*segment, error) {
sc.Lock()
defer sc.Unlock()
start = sc.Standard(start)
@@ -219,7 +218,7 @@ func (sc *segmentController) create(start time.Time, createBlockIfEmpty bool) (*
if err != nil {
return nil, err
}
- return sc.load(start, end, sc.location, createBlockIfEmpty)
+ return sc.load(start, end, sc.location)
}
func (sc *segmentController) sortLst() {
@@ -228,7 +227,8 @@ func (sc *segmentController) sortLst() {
})
}
-func (sc *segmentController) load(start, end time.Time, root string, createBlockIfEmpty bool) (seg *segment, err error) {
+// nolint: contextcheck
+func (sc *segmentController) load(start, end time.Time, root string) (seg *segment, err error) {
suffix := sc.Format(start)
seg, err = openSegment(common.SetPosition(sc.shardCtx, func(p common.Position) common.Position {
p.Segment = suffix
diff --git a/banyand/tsdb/series.go b/banyand/tsdb/series.go
index 386ff91..7ea6e66 100644
--- a/banyand/tsdb/series.go
+++ b/banyand/tsdb/series.go
@@ -88,11 +88,11 @@ type SeriesSpan interface {
var _ Series = (*series)(nil)
type series struct {
- id common.SeriesID
- idLiteral string
blockDB blockDatabase
- shardID common.ShardID
l *logger.Logger
+ idLiteral string
+ id common.SeriesID
+ shardID common.ShardID
}
func (s *series) Get(ctx context.Context, id GlobalItemID) (Item, io.Closer, error) {
@@ -134,7 +134,7 @@ func (s *series) Span(ctx context.Context, timeRange timestamp.TimeRange) (Serie
Str("series", s.idLiteral).
Msg("select series span")
}
- return newSeriesSpan(context.WithValue(context.Background(), logger.ContextKey, l), timeRange, blocks,
+ return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, l), timeRange, blocks,
s.id, s.idLiteral, s.shardID), nil
}
@@ -151,7 +151,7 @@ func (s *series) Create(ctx context.Context, t time.Time) (SeriesSpan, error) {
Str("series", s.idLiteral).
Msg("load a series span")
}
- return newSeriesSpan(context.WithValue(context.Background(), logger.ContextKey, s.l), tr, blocks,
+ return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, s.l), tr, blocks,
s.id, s.idLiteral, s.shardID), nil
}
b, err := s.blockDB.create(ctx, t)
@@ -165,7 +165,7 @@ func (s *series) Create(ctx context.Context, t time.Time) (SeriesSpan, error) {
Str("series", s.idLiteral).
Msg("create a series span")
}
- return newSeriesSpan(context.WithValue(context.Background(), logger.ContextKey, s.l), tr, blocks,
+ return newSeriesSpan(context.WithValue(ctx, logger.ContextKey, s.l), tr, blocks,
s.id, s.idLiteral, s.shardID), nil
}
@@ -188,12 +188,12 @@ func newSeries(ctx context.Context, id common.SeriesID, idLiteral string, blockD
var _ SeriesSpan = (*seriesSpan)(nil)
type seriesSpan struct {
+ l *logger.Logger
+ timeRange timestamp.TimeRange
+ series string
blocks []BlockDelegate
seriesID common.SeriesID
- series string
shardID common.ShardID
- timeRange timestamp.TimeRange
- l *logger.Logger
}
func (s *seriesSpan) Close() (err error) {
diff --git a/banyand/tsdb/series_seek.go b/banyand/tsdb/series_seek.go
index 0ff2196..db66d98 100644
--- a/banyand/tsdb/series_seek.go
+++ b/banyand/tsdb/series_seek.go
@@ -63,13 +63,12 @@ type Seeker interface {
var _ SeekerBuilder = (*seekerBuilder)(nil)
type seekerBuilder struct {
- seriesSpan *seriesSpan
-
predicator index.Filter
- order modelv1.Sort
+ seriesSpan *seriesSpan
indexRuleForSorting *databasev1.IndexRule
- rangeOptsForSorting index.RangeOpts
l *logger.Logger
+ rangeOptsForSorting index.RangeOpts
+ order modelv1.Sort
}
func (s *seekerBuilder) Build() (Seeker, error) {
@@ -109,11 +108,11 @@ func newSeeker(series []Iterator) Seeker {
var _ Item = (*item)(nil)
type item struct {
- itemID common.ItemID
data kv.TimeSeriesReader
- seriesID common.SeriesID
- sortedField []byte
decoderPool encoding.SeriesDecoderPool
+ sortedField []byte
+ itemID common.ItemID
+ seriesID common.SeriesID
}
func (i *item) Time() uint64 {
diff --git a/banyand/tsdb/series_seek_sort.go b/banyand/tsdb/series_seek_sort.go
index 2137899..d56cacf 100644
--- a/banyand/tsdb/series_seek_sort.go
+++ b/banyand/tsdb/series_seek_sort.go
@@ -21,6 +21,7 @@ import (
"sort"
"time"
+ "github.com/pkg/errors"
"go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
@@ -34,7 +35,10 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
-var emptyFilters = make([]filterFn, 0)
+var (
+ ErrUnspecifiedIndexType = errors.New("Unspecified index type")
+ emptyFilters = make([]filterFn, 0)
+)
func (s *seekerBuilder) OrderByIndex(indexRule *databasev1.IndexRule, order modelv1.Sort) SeekerBuilder {
s.indexRuleForSorting = indexRule
@@ -84,6 +88,8 @@ func (s *seekerBuilder) buildSeriesByIndex() (series []Iterator, err error) {
inner, err = b.lsmIndexReader().Iterator(fieldKey, s.rangeOptsForSorting, s.order)
case databasev1.IndexRule_TYPE_INVERTED:
inner, err = b.invertedIndexReader().Iterator(fieldKey, s.rangeOptsForSorting, s.order)
+ case databasev1.IndexRule_TYPE_UNSPECIFIED:
+ return nil, errors.WithMessagef(ErrUnspecifiedIndexType, "index rule:%v", s.indexRuleForSorting)
}
if err != nil {
return nil, err
@@ -159,13 +165,13 @@ var _ Iterator = (*searcherIterator)(nil)
type searcherIterator struct {
fieldIterator index.FieldIterator
- curKey []byte
cur posting.Iterator
data kv.TimeSeriesReader
decoderPool encoding.SeriesDecoderPool
- seriesID common.SeriesID
- filters []filterFn
l *logger.Logger
+ curKey []byte
+ filters []filterFn
+ seriesID common.SeriesID
}
func (s *searcherIterator) Next() bool {
@@ -179,7 +185,6 @@ func (s *searcherIterator) Next() bool {
}
}
if s.cur.Next() {
-
for _, filter := range s.filters {
if !filter(s.Val()) {
return s.Next()
@@ -225,8 +230,8 @@ var _ Iterator = (*mergedIterator)(nil)
type mergedIterator struct {
curr Iterator
- index int
delegated []Iterator
+ index int
}
func (m *mergedIterator) Next() bool {
diff --git a/banyand/tsdb/series_write.go b/banyand/tsdb/series_write.go
index 661d1f7..b6da92c 100644
--- a/banyand/tsdb/series_write.go
+++ b/banyand/tsdb/series_write.go
@@ -98,7 +98,7 @@ func (w *writerBuilder) Build() (Writer, error) {
return nil, errors.WithStack(ErrNoVal)
}
for i, value := range w.values {
- for j := i + 1; j < len(w.values); j = j + 1 {
+ for j := i + 1; j < len(w.values); j++ {
if value.family == nil && w.values[j].family == nil {
return nil, errors.Wrap(ErrDuplicatedFamily, "default family")
}
@@ -132,13 +132,13 @@ func newWriterBuilder(seriesSpan *seriesSpan) WriterBuilder {
var _ Writer = (*writer)(nil)
type writer struct {
- block BlockDelegate
ts time.Time
+ block BlockDelegate
+ itemID *GlobalItemID
columns []struct {
family []byte
val []byte
}
- itemID *GlobalItemID
}
func (w *writer) ItemID() GlobalItemID {
@@ -173,8 +173,8 @@ func (w *writer) String() string {
}
type dataBucket struct {
- seriesID common.SeriesID
family []byte
+ seriesID common.SeriesID
}
func (d dataBucket) marshal() []byte {
diff --git a/banyand/tsdb/seriesdb.go b/banyand/tsdb/seriesdb.go
index c9ab074..aac8e55 100644
--- a/banyand/tsdb/seriesdb.go
+++ b/banyand/tsdb/seriesdb.go
@@ -75,9 +75,9 @@ func (e Entity) Copy() Entity {
return a
}
-func NewEntity(len int) Entity {
- e := make(Entity, len)
- for i := 0; i < len; i++ {
+func NewEntity(length int) Entity {
+ e := make(Entity, length)
+ for i := 0; i < length; i++ {
e[i] = AnyEntry
}
return e
@@ -244,12 +244,11 @@ var (
)
type seriesDB struct {
- sync.Mutex
- l *logger.Logger
-
- segCtrl *segmentController
seriesMetadata kv.Store
- sID common.ShardID
+ l *logger.Logger
+ segCtrl *segmentController
+ sync.Mutex
+ sID common.ShardID
}
func (s *seriesDB) GetByID(id common.SeriesID) (Series, error) {
@@ -352,7 +351,7 @@ func (s *seriesDB) List(ctx context.Context, path Path) (SeriesList, error) {
l := logger.FetchOrDefault(ctx, "series_database", s.l)
if path.isFull {
data, err := s.seriesMetadata.Get(prefix)
- if err != nil && err != kv.ErrKeyNotFound {
+ if err != nil && errors.Is(err, kv.ErrKeyNotFound) {
return nil, err
}
if err == nil {
@@ -368,7 +367,7 @@ func (s *seriesDB) List(ctx context.Context, path Path) (SeriesList, error) {
Uint64("series_id", uint64(seriesID)).
Msg("got a series with a full path")
}
- return []Series{newSeries(s.context(), seriesID, series, s)}, nil
+ return []Series{newSeries(ctx, seriesID, series, s)}, nil
}
if e := l.Debug(); e.Enabled() {
e.Hex("path", path.prefix).Msg("doesn't get any series")
@@ -446,7 +445,7 @@ func (s *seriesDB) create(ctx context.Context, ts time.Time) (BlockDelegate, err
}
return block.delegate(ctx)
}
- seg, err := s.segCtrl.create(timeRange.Start, false)
+ seg, err := s.segCtrl.create(timeRange.Start)
if err != nil {
return nil, err
}
@@ -500,7 +499,7 @@ func newSeriesDataBase(ctx context.Context, shardID common.ShardID, path string,
// HashEntity runs hash function (e.g. with xxhash algorithm) on each segment of the Entity,
// and concatenates all uint64 in byte array. So the return length of the byte array will be
-// 8 (every uint64 has 8 bytes) * length of the input
+// 8 (every uint64 has 8 bytes) * length of the input.
func HashEntity(entity Entity) []byte {
result := make([]byte, 0, len(entity)*8)
for _, entry := range entity {
@@ -557,7 +556,6 @@ func (a SeriesList) Merge(other SeriesList) SeriesList {
}
final = append(final, other[j])
j++
-
}
}
for ; i < len(a); i++ {
diff --git a/banyand/tsdb/seriesdb_test.go b/banyand/tsdb/seriesdb_test.go
index 96d9f82..4d232f5 100644
--- a/banyand/tsdb/seriesdb_test.go
+++ b/banyand/tsdb/seriesdb_test.go
@@ -422,9 +422,9 @@ func Test_SeriesDatabase_List(t *testing.T) {
data := setUpEntities(tester, s)
tests := []struct {
name string
+ want SeriesList
path Path
wantErr bool
- want SeriesList
}{
{
name: "equal",
@@ -504,8 +504,8 @@ func Test_SeriesDatabase_List(t *testing.T) {
}
type entityWithID struct {
- id common.SeriesID
entity Entity
+ id common.SeriesID
}
func setUpEntities(t *assert.Assertions, db SeriesDatabase) []*entityWithID {
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index 23dabb3..f89b2d8 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -43,17 +43,15 @@ const (
var _ Shard = (*shard)(nil)
type shard struct {
- l *logger.Logger
- id common.ShardID
- position common.Position
-
seriesDatabase SeriesDatabase
indexDatabase IndexDatabase
+ l *logger.Logger
segmentController *segmentController
segmentManageStrategy *bucket.Strategy
scheduler *timestamp.Scheduler
-
- closeOnce sync.Once
+ position common.Position
+ closeOnce sync.Once
+ id common.ShardID
}
func OpenShard(ctx context.Context, id common.ShardID,
@@ -98,10 +96,7 @@ func OpenShard(ctx context.Context, id common.ShardID,
return nil, err
}
s.seriesDatabase = sdb
- idb, err := newIndexDatabase(shardCtx, s.id, s.segmentController)
- if err != nil {
- return nil, err
- }
+ idb := newIndexDatabase(shardCtx, s.id, s.segmentController)
s.indexDatabase = idb
s.segmentManageStrategy, err = bucket.NewStrategy(s.segmentController, bucket.WithLogger(s.l))
if err != nil {
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index 770c5bd..70546db 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -85,16 +85,16 @@ type Shard interface {
var _ Database = (*database)(nil)
type DatabaseOpts struct {
- Location string
- ShardNum uint32
EncodingMethod EncodingMethod
+ Location string
SegmentInterval IntervalRule
BlockInterval IntervalRule
TTL IntervalRule
BlockMemSize int64
SeriesMemSize int64
- EnableGlobalIndex bool
GlobalIndexMemSize int64
+ ShardNum uint32
+ EnableGlobalIndex bool
}
type EncodingMethod struct {
@@ -132,8 +132,8 @@ func readSectionID(data []byte, offset int) (SectionID, int) {
}
type BlockState struct {
- ID BlockID
TimeRange timestamp.TimeRange
+ ID BlockID
Closed bool
}
type ShardState struct {
@@ -145,13 +145,12 @@ type ShardState struct {
type database struct {
logger *logger.Logger
location string
- shardNum uint32
+ sLst []Shard
segmentSize IntervalRule
blockSize IntervalRule
ttl IntervalRule
-
- sLst []Shard
sync.Mutex
+ shardNum uint32
}
func (d *database) Shards() []Shard {
@@ -278,7 +277,6 @@ func loadDatabase(ctx context.Context, db *database) (Database, error) {
if err != nil {
return nil, errors.WithMessage(err, "load the database failed")
}
-
}
return db, nil
}
diff --git a/bydbctl/cmd/bydbctl/main.go b/bydbctl/cmd/bydbctl/main.go
index 6cbbabc..557ef14 100644
--- a/bydbctl/cmd/bydbctl/main.go
+++ b/bydbctl/cmd/bydbctl/main.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package main provides main entry for the command-line toolkit, i.e. bydbctl
+// Package main provides main entry for the command-line toolkit, i.e. bydbctl.
package main
import (
diff --git a/bydbctl/internal/cmd/group.go b/bydbctl/internal/cmd/group.go
index 60d1f1d..f2c32fd 100644
--- a/bydbctl/internal/cmd/group.go
+++ b/bydbctl/internal/cmd/group.go
@@ -24,8 +24,8 @@ import (
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
- common_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -43,12 +43,12 @@ func newGroupCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- g := new(common_v1.Group)
+ g := new(commonv1.Group)
err := protojson.Unmarshal(request.data, g)
if err != nil {
return nil, err
}
- cr := &database_v1.GroupRegistryServiceCreateRequest{
+ cr := &databasev1.GroupRegistryServiceCreateRequest{
Group: g,
}
b, err := protojson.Marshal(cr)
@@ -72,12 +72,12 @@ func newGroupCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- g := new(common_v1.Group)
+ g := new(commonv1.Group)
err := protojson.Unmarshal(request.data, g)
if err != nil {
return nil, err
}
- cr := &database_v1.GroupRegistryServiceUpdateRequest{
+ cr := &databasev1.GroupRegistryServiceUpdateRequest{
Group: g,
}
b, err := protojson.Marshal(cr)
diff --git a/bydbctl/internal/cmd/group_test.go b/bydbctl/internal/cmd/group_test.go
index 53c8756..5a84abc 100644
--- a/bydbctl/internal/cmd/group_test.go
+++ b/bydbctl/internal/cmd/group_test.go
@@ -25,7 +25,7 @@ import (
"github.com/spf13/cobra"
"github.com/zenizh/go-capturer"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -72,7 +72,7 @@ resource_opts:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.GroupRegistryServiceGetResponse)
+ resp := new(databasev1.GroupRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Group.Metadata.Name).To(Equal("group1"))
})
@@ -139,7 +139,7 @@ resource_opts:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.GroupRegistryServiceListResponse)
+ resp := new(databasev1.GroupRegistryServiceListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Group).To(HaveLen(4))
})
diff --git a/bydbctl/internal/cmd/index_rule.go b/bydbctl/internal/cmd/index_rule.go
index aea2088..1989ef3 100644
--- a/bydbctl/internal/cmd/index_rule.go
+++ b/bydbctl/internal/cmd/index_rule.go
@@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -46,12 +46,12 @@ func newIndexRuleCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.IndexRule)
+ s := new(databasev1.IndexRule)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.IndexRuleRegistryServiceCreateRequest{
+ cr := &databasev1.IndexRuleRegistryServiceCreateRequest{
IndexRule: s,
}
b, err := protojson.Marshal(cr)
@@ -75,12 +75,12 @@ func newIndexRuleCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.IndexRule)
+ s := new(databasev1.IndexRule)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.IndexRuleRegistryServiceUpdateRequest{
+ cr := &databasev1.IndexRuleRegistryServiceUpdateRequest{
IndexRule: s,
}
b, err := protojson.Marshal(cr)
diff --git a/bydbctl/internal/cmd/index_rule_binding.go b/bydbctl/internal/cmd/index_rule_binding.go
index 5c7bcb5..a9bf2a5 100644
--- a/bydbctl/internal/cmd/index_rule_binding.go
+++ b/bydbctl/internal/cmd/index_rule_binding.go
@@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -46,12 +46,12 @@ func newIndexRuleBindingCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.IndexRuleBinding)
+ s := new(databasev1.IndexRuleBinding)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.IndexRuleBindingRegistryServiceCreateRequest{
+ cr := &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
IndexRuleBinding: s,
}
b, err := protojson.Marshal(cr)
@@ -75,12 +75,12 @@ func newIndexRuleBindingCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.IndexRuleBinding)
+ s := new(databasev1.IndexRuleBinding)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.IndexRuleBindingRegistryServiceUpdateRequest{
+ cr := &databasev1.IndexRuleBindingRegistryServiceUpdateRequest{
IndexRuleBinding: s,
}
b, err := protojson.Marshal(cr)
diff --git a/bydbctl/internal/cmd/index_rule_binding_test.go b/bydbctl/internal/cmd/index_rule_binding_test.go
index 9de2cb6..99ac03d 100644
--- a/bydbctl/internal/cmd/index_rule_binding_test.go
+++ b/bydbctl/internal/cmd/index_rule_binding_test.go
@@ -25,7 +25,7 @@ import (
"github.com/spf13/cobra"
"github.com/zenizh/go-capturer"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -94,7 +94,7 @@ subject:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(database_v1.IndexRuleBindingRegistryServiceGetResponse)
+ resp := new(databasev1.IndexRuleBindingRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRuleBinding.Metadata.Group).To(Equal("group1"))
Expect(resp.IndexRuleBinding.Metadata.Name).To(Equal("name1"))
@@ -120,7 +120,7 @@ subject:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.IndexRuleBindingRegistryServiceGetResponse)
+ resp := new(databasev1.IndexRuleBindingRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRuleBinding.Metadata.Group).To(Equal("group1"))
Expect(resp.IndexRuleBinding.Metadata.Name).To(Equal("name1"))
@@ -162,7 +162,7 @@ subject:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.IndexRuleBindingRegistryServiceListResponse)
+ resp := new(databasev1.IndexRuleBindingRegistryServiceListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRuleBinding).To(HaveLen(2))
})
diff --git a/bydbctl/internal/cmd/index_rule_test.go b/bydbctl/internal/cmd/index_rule_test.go
index f71b2f0..ede7edb 100644
--- a/bydbctl/internal/cmd/index_rule_test.go
+++ b/bydbctl/internal/cmd/index_rule_test.go
@@ -25,7 +25,7 @@ import (
"github.com/spf13/cobra"
"github.com/zenizh/go-capturer"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -91,7 +91,7 @@ metadata:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(database_v1.IndexRuleRegistryServiceGetResponse)
+ resp := new(databasev1.IndexRuleRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRule.Metadata.Group).To(Equal("group1"))
Expect(resp.IndexRule.Metadata.Name).To(Equal("name1"))
@@ -113,7 +113,7 @@ metadata:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.IndexRuleRegistryServiceGetResponse)
+ resp := new(databasev1.IndexRuleRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRule.Metadata.Group).To(Equal("group1"))
Expect(resp.IndexRule.Metadata.Name).To(Equal("name1"))
@@ -151,7 +151,7 @@ metadata:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.IndexRuleRegistryServiceListResponse)
+ resp := new(databasev1.IndexRuleRegistryServiceListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.IndexRule).To(HaveLen(2))
})
diff --git a/bydbctl/internal/cmd/measure.go b/bydbctl/internal/cmd/measure.go
index 178780c..4be794e 100644
--- a/bydbctl/internal/cmd/measure.go
+++ b/bydbctl/internal/cmd/measure.go
@@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -46,12 +46,12 @@ func newMeasureCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.Measure)
+ s := new(databasev1.Measure)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.MeasureRegistryServiceCreateRequest{
+ cr := &databasev1.MeasureRegistryServiceCreateRequest{
Measure: s,
}
b, err := protojson.Marshal(cr)
@@ -75,12 +75,12 @@ func newMeasureCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.Measure)
+ s := new(databasev1.Measure)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.MeasureRegistryServiceUpdateRequest{
+ cr := &databasev1.MeasureRegistryServiceUpdateRequest{
Measure: s,
}
b, err := protojson.Marshal(cr)
diff --git a/bydbctl/internal/cmd/measure_test.go b/bydbctl/internal/cmd/measure_test.go
index 3327069..28c8334 100644
--- a/bydbctl/internal/cmd/measure_test.go
+++ b/bydbctl/internal/cmd/measure_test.go
@@ -29,8 +29,8 @@ import (
grpclib "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measure_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -97,7 +97,7 @@ metadata:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.MeasureRegistryServiceGetResponse)
+ resp := new(databasev1.MeasureRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Measure.Metadata.Group).To(Equal("group1"))
Expect(resp.Measure.Metadata.Name).To(Equal("name1"))
@@ -121,7 +121,7 @@ entity:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.MeasureRegistryServiceGetResponse)
+ resp := new(databasev1.MeasureRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Measure.Metadata.Group).To(Equal("group1"))
Expect(resp.Measure.Metadata.Name).To(Equal("name1"))
@@ -160,7 +160,7 @@ metadata:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.MeasureRegistryServiceListResponse)
+ resp := new(databasev1.MeasureRegistryServiceListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Measure).To(HaveLen(2))
})
@@ -219,7 +219,7 @@ tagProjection:
Eventually(func() int {
out := issue()
GinkgoWriter.Println(out)
- resp := new(measure_v1.QueryResponse)
+ resp := new(measurev1.QueryResponse)
helpers.UnmarshalYAML([]byte(out), resp)
GinkgoWriter.Println(resp)
return len(resp.DataPoints)
@@ -257,7 +257,7 @@ tagProjection:
Eventually(issue, flags.EventuallyTimeout).ShouldNot(ContainSubstring("code:"))
Eventually(func() int {
out := issue()
- resp := new(measure_v1.QueryResponse)
+ resp := new(measurev1.QueryResponse)
helpers.UnmarshalYAML([]byte(out), resp)
GinkgoWriter.Println(resp)
return len(resp.DataPoints)
diff --git a/bydbctl/internal/cmd/property.go b/bydbctl/internal/cmd/property.go
index b47b823..47fa68f 100644
--- a/bydbctl/internal/cmd/property.go
+++ b/bydbctl/internal/cmd/property.go
@@ -22,7 +22,7 @@ import (
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
- property_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -51,13 +51,14 @@ func newPropertyCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
return rest(func() ([]reqBody, error) { return parseFromYAMLForProperty(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(property_v1.Property)
+ s := new(propertyv1.Property)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &property_v1.ApplyRequest{
+ cr := &propertyv1.ApplyRequest{
Property: s,
+ Strategy: propertyv1.ApplyRequest_STRATEGY_MERGE,
}
b, err := protojson.Marshal(cr)
if err != nil {
diff --git a/bydbctl/internal/cmd/property_test.go b/bydbctl/internal/cmd/property_test.go
index f538dc2..95ac1f9 100644
--- a/bydbctl/internal/cmd/property_test.go
+++ b/bydbctl/internal/cmd/property_test.go
@@ -27,7 +27,7 @@ import (
"github.com/zenizh/go-capturer"
"google.golang.org/protobuf/testing/protocmp"
- property_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -70,9 +70,9 @@ tags:
int:
value: 3
`
- p1Proto := new(property_v1.Property)
+ p1Proto := new(propertyv1.Property)
helpers.UnmarshalYAML([]byte(p1YAML), p1Proto)
- p2Proto := new(property_v1.Property)
+ p2Proto := new(propertyv1.Property)
helpers.UnmarshalYAML([]byte(p2YAML), p2Proto)
BeforeEach(func() {
_, addr, deferFunc = setup.SetUp()
@@ -99,7 +99,7 @@ tags:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(property_v1.GetResponse)
+ resp := new(propertyv1.GetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(cmp.Equal(resp.Property, p1Proto,
protocmp.IgnoreUnknown(),
@@ -117,7 +117,7 @@ tags:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(property_v1.GetResponse)
+ resp := new(propertyv1.GetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Property.Tags).To(HaveLen(1))
Expect(resp.Property.Tags[0].Key).To(Equal("state"))
@@ -133,7 +133,7 @@ tags:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(property_v1.GetResponse)
+ resp := new(propertyv1.GetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(cmp.Equal(resp.Property, p1Proto,
protocmp.IgnoreUnknown(),
@@ -165,7 +165,7 @@ tags:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(property_v1.GetResponse)
+ resp := new(propertyv1.GetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(cmp.Equal(resp.Property, p2Proto,
@@ -219,7 +219,7 @@ tags:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(property_v1.ListResponse)
+ resp := new(propertyv1.ListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Property).To(HaveLen(2))
})
diff --git a/bydbctl/internal/cmd/root.go b/bydbctl/internal/cmd/root.go
index c9ea5c7..7c7a16e 100644
--- a/bydbctl/internal/cmd/root.go
+++ b/bydbctl/internal/cmd/root.go
@@ -15,10 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-// Package cmd is an internal package defining cli commands for bydbctl
+// Package cmd is an internal package defining cli commands for bydbctl.
package cmd
import (
+ "errors"
"fmt"
"os"
@@ -90,7 +91,7 @@ func initConfig() {
}
if err := readCfg(); err != nil {
- if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+ if !errors.As(err, &viper.ConfigFileNotFoundError{}) {
cobra.CheckErr(err)
}
cobra.CheckErr(viper.SafeWriteConfig())
diff --git a/bydbctl/internal/cmd/stream.go b/bydbctl/internal/cmd/stream.go
index 75d894f..803cb05 100644
--- a/bydbctl/internal/cmd/stream.go
+++ b/bydbctl/internal/cmd/stream.go
@@ -25,7 +25,7 @@ import (
"github.com/spf13/viper"
"google.golang.org/protobuf/encoding/protojson"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/version"
)
@@ -51,12 +51,12 @@ func newStreamCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.Stream)
+ s := new(databasev1.Stream)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.StreamRegistryServiceCreateRequest{
+ cr := &databasev1.StreamRegistryServiceCreateRequest{
Stream: s,
}
b, err := protojson.Marshal(cr)
@@ -80,12 +80,12 @@ func newStreamCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) (err error) {
return rest(func() ([]reqBody, error) { return parseNameAndGroupFromYAML(cmd.InOrStdin()) },
func(request request) (*resty.Response, error) {
- s := new(database_v1.Stream)
+ s := new(databasev1.Stream)
err := protojson.Unmarshal(request.data, s)
if err != nil {
return nil, err
}
- cr := &database_v1.StreamRegistryServiceUpdateRequest{
+ cr := &databasev1.StreamRegistryServiceUpdateRequest{
Stream: s,
}
b, err := protojson.Marshal(cr)
diff --git a/bydbctl/internal/cmd/stream_test.go b/bydbctl/internal/cmd/stream_test.go
index 4154f7b..1790952 100644
--- a/bydbctl/internal/cmd/stream_test.go
+++ b/bydbctl/internal/cmd/stream_test.go
@@ -29,8 +29,8 @@ import (
grpclib "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- stream_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/bydbctl/internal/cmd"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -103,7 +103,7 @@ tagFamilies:
Expect(err).NotTo(HaveOccurred())
})
GinkgoWriter.Println(out)
- resp := new(database_v1.StreamRegistryServiceGetResponse)
+ resp := new(databasev1.StreamRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Stream.Metadata.Group).To(Equal("group1"))
Expect(resp.Stream.Metadata.Name).To(Equal("name1"))
@@ -127,7 +127,7 @@ entity:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.StreamRegistryServiceGetResponse)
+ resp := new(databasev1.StreamRegistryServiceGetResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Stream.Metadata.Group).To(Equal("group1"))
Expect(resp.Stream.Metadata.Name).To(Equal("name1"))
@@ -166,7 +166,7 @@ metadata:
err := rootCmd.Execute()
Expect(err).NotTo(HaveOccurred())
})
- resp := new(database_v1.StreamRegistryServiceListResponse)
+ resp := new(databasev1.StreamRegistryServiceListResponse)
helpers.UnmarshalYAML([]byte(out), resp)
Expect(resp.Stream).To(HaveLen(2))
})
@@ -225,7 +225,7 @@ projection:
Eventually(issue, flags.EventuallyTimeout).ShouldNot(ContainSubstring("code:"))
Eventually(func() int {
out := issue()
- resp := new(stream_v1.QueryResponse)
+ resp := new(streamv1.QueryResponse)
helpers.UnmarshalYAML([]byte(out), resp)
GinkgoWriter.Println(resp)
return len(resp.Elements)
@@ -263,7 +263,7 @@ projection:
Eventually(issue, flags.EventuallyTimeout).ShouldNot(ContainSubstring("code:"))
Eventually(func() int {
out := issue()
- resp := new(stream_v1.QueryResponse)
+ resp := new(streamv1.QueryResponse)
helpers.UnmarshalYAML([]byte(out), resp)
GinkgoWriter.Println(resp)
return len(resp.Elements)
diff --git a/bydbctl/pkg/file/read.go b/bydbctl/pkg/file/read.go
index f4708b1..8f05294 100644
--- a/bydbctl/pkg/file/read.go
+++ b/bydbctl/pkg/file/read.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package file provides utils to handle files
+// Package file provides utils to handle files.
package file
import (
diff --git a/deprecated-golangci.yml b/deprecated-golangci.yml
deleted file mode 100644
index 61c209d..0000000
--- a/deprecated-golangci.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-linters:
- enable:
- - deadcode
- - errcheck
- - goconst
- - golint
- - ineffassign
- - lll
- - misspell
- - structcheck
- - unconvert
- - varcheck
- - govet
- - gci
- - prealloc
- - unused
- - staticcheck
- - gosimple
- - megacheck
- disable:
- - interfacer
-linters-settings:
- lll:
- line-length: 170
- goconst:
- min-occurrences: 4
- govet:
- check-shadowing: true
- gci:
- sections:
- - standard
- - default
- - prefix(github.com/apache/skywalking-banyandb/)
-run:
- deadline: 10m
- skip-files:
- - ".*\\.pb\\.validate\\.go"
-issues:
- exclude:
- # staticcheck
- - 'SA1019: Package github.com/golang/protobuf/proto is deprecated: Use the "google.golang.org/protobuf/proto" package instead.'
- - 'SA1019: Package github.com/golang/protobuf/jsonpb is deprecated: Use the "google.golang.org/protobuf/encoding/protojson" package instead.'
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 2c5839b..799d11a 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -3,7 +3,7 @@
## Table of Contents
-- [banyandb/common/v1/common.proto](#banyandb_common_v1_common-proto)
+- [banyandb/common/v1/common.proto](#banyandb_commonv1_common-proto)
- [Group](#banyandb-common-v1-Group)
- [IntervalRule](#banyandb-common-v1-IntervalRule)
- [Metadata](#banyandb-common-v1-Metadata)
@@ -212,7 +212,7 @@
-<a name="banyandb_common_v1_common-proto"></a>
+<a name="banyandb_commonv1_common-proto"></a>
<p align="right"><a href="#top">Top</a></p>
## banyandb/common/v1/common.proto
diff --git a/pkg/Makefile b/pkg/Makefile
index a133ae0..043bc21 100644
--- a/pkg/Makefile
+++ b/pkg/Makefile
@@ -21,5 +21,5 @@ NAME := pkg
include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
include ../scripts/build/test.mk
-include ../scripts/build/lint-deprecated.mk
+include ../scripts/build/lint.mk
include ../scripts/build/help.mk
diff --git a/pkg/bit/reader.go b/pkg/bit/reader.go
index 22cb929..6174728 100644
--- a/pkg/bit/reader.go
+++ b/pkg/bit/reader.go
@@ -21,21 +21,21 @@ import (
"io"
)
-// Reader reads bits from buffer
+// Reader reads bits from buffer.
type Reader struct {
in io.ByteReader
cache byte
len byte
}
-// NewReader crate bit reader
+// NewReader crate bit reader.
func NewReader(in io.ByteReader) *Reader {
return &Reader{
in: in,
}
}
-// ReadBool reads a bit, 1 returns true, 0 returns false
+// ReadBool reads a bit, 1 returns true, 0 returns false.
func (r *Reader) ReadBool() (bool, error) {
if r.len == 0 {
b, err := r.in.ReadByte()
@@ -51,7 +51,7 @@ func (r *Reader) ReadBool() (bool, error) {
return b != 0, nil
}
-// ReadBits read number of bits
+// ReadBits read number of bits.
func (r *Reader) ReadBits(numBits int) (uint64, error) {
var result uint64
@@ -78,7 +78,7 @@ func (r *Reader) ReadBits(numBits int) (uint64, error) {
return result, nil
}
-// ReadByte reads a byte
+// ReadByte reads a byte.
func (r *Reader) ReadByte() (byte, error) {
if r.len == 0 {
b, err := r.in.ReadByte()
@@ -97,7 +97,7 @@ func (r *Reader) ReadByte() (byte, error) {
return result, nil
}
-// Reset resets the reader to read from a new slice
+// Reset resets the reader to read from a new slice.
func (r *Reader) Reset() {
r.len = 0
r.cache = 0
diff --git a/pkg/bit/writer.go b/pkg/bit/writer.go
index 9995f01..a7ed564 100644
--- a/pkg/bit/writer.go
+++ b/pkg/bit/writer.go
@@ -21,21 +21,21 @@ import (
"bytes"
)
-// Writer writes bits to an io.BufferWriter
+// Writer writes bits to an io.BufferWriter.
type Writer struct {
out *bytes.Buffer
cache byte
available byte
}
-// NewWriter create bit writer
+// NewWriter create bit writer.
func NewWriter(buffer *bytes.Buffer) *Writer {
bw := new(Writer)
bw.Reset(buffer)
return bw
}
-// Reset writes to a new writer
+// Reset writes to a new writer.
func (w *Writer) Reset(buffer *bytes.Buffer) {
if buffer == nil {
w.out.Reset()
@@ -64,13 +64,13 @@ func (w *Writer) WriteBool(b bool) {
}
}
-// WriteBits writes number of bits
+// WriteBits writes number of bits.
func (w *Writer) WriteBits(u uint64, numBits int) {
u <<= 64 - uint(numBits)
for ; numBits >= 8; numBits -= 8 {
byt := byte(u >> 56)
- w.WriteByte(byt)
+ _ = w.WriteByte(byt)
u <<= 8
}
@@ -81,13 +81,16 @@ func (w *Writer) WriteBits(u uint64, numBits int) {
}
}
-// WriteByte write a byte
-func (w *Writer) WriteByte(b byte) {
- _ = w.out.WriteByte(w.cache | (b >> (8 - w.available)))
+// WriteByte write a byte.
+func (w *Writer) WriteByte(b byte) error {
+ if err := w.out.WriteByte(w.cache | (b >> (8 - w.available))); err != nil {
+ return err
+ }
w.cache = b << w.available
+ return nil
}
-// Flush flushes the currently in-process byte
+// Flush flushes the currently in-process byte.
func (w *Writer) Flush() {
if w.available != 8 {
_ = w.out.WriteByte(w.cache)
diff --git a/pkg/buffer/writer.go b/pkg/buffer/writer.go
index e109d25..3d87a7b 100644
--- a/pkg/buffer/writer.go
+++ b/pkg/buffer/writer.go
@@ -22,7 +22,7 @@ import (
"encoding/binary"
)
-// Writer writes data into a buffer
+// Writer writes data into a buffer.
type Writer struct {
buf *bytes.Buffer
diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go
index 75d66c2..90a4bc3 100644
--- a/pkg/bus/bus.go
+++ b/pkg/bus/bus.go
@@ -27,7 +27,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-// Payload represents a simple data
+// Payload represents a simple data.
type Payload interface{}
type (
MessageID uint64
@@ -37,10 +37,10 @@ type (
}
)
-// Message is send on the bus to all subscribed listeners
+// Message is send on the bus to all subscribed listeners.
type Message struct {
- id MessageID
payload Payload
+ id MessageID
}
func (m Message) ID() MessageID {
@@ -82,19 +82,19 @@ type Topic struct {
Type ChType
}
-func UniTopic(ID string) Topic {
- return Topic{ID: ID, Type: ChTypeUnidirectional}
+func UniTopic(id string) Topic {
+ return Topic{ID: id, Type: ChTypeUnidirectional}
}
-func BiTopic(ID string) Topic {
- return Topic{ID: ID, Type: ChTypeBidirectional}
+func BiTopic(id string) Topic {
+ return Topic{ID: id, Type: ChTypeBidirectional}
}
-// The Bus allows publish-subscribe-style communication between components
+// The Bus allows publish-subscribe-style communication between components.
type Bus struct {
topics map[Topic][]Channel
- mutex sync.RWMutex
closer *run.Closer
+ mutex sync.RWMutex
}
func NewBus() *Bus {
@@ -132,7 +132,7 @@ func (l *localFuture) Get() (Message, error) {
}
m, ok := <-l.retCh
if ok {
- l.retCount = l.retCount - 1
+ l.retCount--
return m, nil
}
return Message{}, io.EOF
@@ -143,7 +143,7 @@ func (l *localFuture) GetAll() ([]Message, error) {
ret := make([]Message, 0, l.retCount)
for {
m, err := l.Get()
- if err == io.EOF {
+ if errors.Is(err, io.EOF) {
return ret, globalErr
}
if err != nil {
@@ -155,8 +155,8 @@ func (l *localFuture) GetAll() ([]Message, error) {
}
type Event struct {
- m Message
f Future
+ m Message
}
func (b *Bus) Publish(topic Topic, message ...Message) (Future, error) {
diff --git a/pkg/bus/bus_test.go b/pkg/bus/bus_test.go
index c3c4357..156e6f7 100644
--- a/pkg/bus/bus_test.go
+++ b/pkg/bus/bus_test.go
@@ -18,6 +18,7 @@
package bus
import (
+ "errors"
"reflect"
"sort"
"sync"
@@ -29,14 +30,14 @@ func TestBus_PubAndSub(t *testing.T) {
type message struct {
topic Topic
messageIDS []MessageID
- wantErr bool
wantRet []MessageID
+ wantErr bool
}
type listener struct {
wantTopic Topic
wantMessages []MessageID
- wantErr bool
ret []MessageID
+ wantErr bool
}
tests := []struct {
name string
@@ -184,7 +185,7 @@ func TestBus_PubAndSub(t *testing.T) {
}
go func(want []MessageID) {
ret, errRet := f.GetAll()
- if errRet == ErrEmptyFuture {
+ if errors.Is(errRet, ErrEmptyFuture) {
return
} else if errRet != nil {
t.Errorf("Publish()'s return message error = %v", err)
@@ -232,10 +233,10 @@ func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
var _ MessageListener = new(mockListener)
type mockListener struct {
- queue []MessageID
wg *sync.WaitGroup
closeWg *sync.WaitGroup
ret chan Message
+ queue []MessageID
}
func (m *mockListener) Rev(message Message) Message {
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 1c71ce2..0ab6d9b 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -18,6 +18,7 @@
package config
import (
+ "errors"
"fmt"
"strings"
@@ -32,8 +33,8 @@ const (
)
type config struct {
- name string
viper *viper.Viper
+ name string
}
func Load(name string, fs *pflag.FlagSet) error {
@@ -62,7 +63,7 @@ func (c *config) initializeConfig(fs *pflag.FlagSet) error {
// if we cannot parse the config file.
if err := v.ReadInConfig(); err != nil {
// It's okay if there isn't a config file
- if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+ if !errors.As(err, &viper.ConfigFileNotFoundError{}) {
return err
}
}
@@ -82,7 +83,7 @@ func (c *config) initializeConfig(fs *pflag.FlagSet) error {
return bindFlags(fs, v)
}
-// Bind each cobra flag to its associated viper configuration (config file and environment variable)
+// Bind each cobra flag to its associated viper configuration (config file and environment variable).
func bindFlags(fs *pflag.FlagSet, v *viper.Viper) error {
var err error
fs.VisitAll(func(f *pflag.Flag) {
diff --git a/pkg/convert/number.go b/pkg/convert/number.go
index bedc5c0..9704d57 100644
--- a/pkg/convert/number.go
+++ b/pkg/convert/number.go
@@ -34,7 +34,7 @@ func Int64ToBytes(i int64) []byte {
}
u := uint64(abs)
if i >= 0 {
- u = u | 1<<63
+ u |= 1 << 63
} else {
u = 1<<63 - u
}
@@ -56,7 +56,7 @@ func Uint32ToBytes(u uint32) []byte {
func BytesToInt64(b []byte) int64 {
u := binary.BigEndian.Uint64(b)
if b[0] >= 128 {
- u = u ^ 1<<63
+ u ^= 1 << 63
} else {
u = 1<<63 - u
}
diff --git a/pkg/encoding/encoding.go b/pkg/encoding/encoding.go
index efc1f6e..67f3b68 100644
--- a/pkg/encoding/encoding.go
+++ b/pkg/encoding/encoding.go
@@ -48,7 +48,7 @@ type SeriesEncoderPool interface {
Put(encoder SeriesEncoder)
}
-// SeriesEncoder encodes time series data point
+// SeriesEncoder encodes time series data point.
type SeriesEncoder interface {
// Append a data point
Append(ts uint64, value []byte)
@@ -67,7 +67,7 @@ type SeriesDecoderPool interface {
Put(encoder SeriesDecoder)
}
-// SeriesDecoder decodes encoded time series data
+// SeriesDecoder decodes encoded time series data.
type SeriesDecoder interface {
// Decode the time series data
Decode(key, data []byte) error
@@ -83,7 +83,7 @@ type SeriesDecoder interface {
Range() (start, end uint64)
}
-// SeriesIterator iterates time series data
+// SeriesIterator iterates time series data.
type SeriesIterator interface {
// Next scroll the cursor to the next
Next() bool
diff --git a/pkg/encoding/int.go b/pkg/encoding/int.go
index c9a48a4..3b8522c 100644
--- a/pkg/encoding/int.go
+++ b/pkg/encoding/int.go
@@ -42,10 +42,10 @@ var (
)
type intEncoderPoolDelegator struct {
- name string
pool *sync.Pool
- size int
fn ParseInterval
+ name string
+ size int
}
func NewIntEncoderPool(name string, size int, fn ParseInterval) SeriesEncoderPool {
@@ -109,11 +109,11 @@ var _ SeriesEncoder = (*intEncoder)(nil)
type ParseInterval = func(key []byte) time.Duration
type intEncoder struct {
- name string
buff *bytes.Buffer
bw *bit.Writer
values *XOREncoder
fn ParseInterval
+ name string
interval time.Duration
startTime uint64
prevTime uint64
@@ -189,13 +189,13 @@ func (ie *intEncoder) StartTime() uint64 {
var _ SeriesDecoder = (*intDecoder)(nil)
type intDecoder struct {
- name string
fn ParseInterval
+ name string
+ area []byte
size int
interval time.Duration
startTime uint64
num int
- area []byte
}
func (i *intDecoder) Decode(key, data []byte) error {
@@ -248,16 +248,15 @@ var (
)
type intIterator struct {
+ err error
+ br *bit.Reader
+ values *XORDecoder
endTime uint64
interval int
size int
- br *bit.Reader
- values *XORDecoder
-
currVal uint64
currTime uint64
index int
- err error
}
func (i *intIterator) Next() bool {
diff --git a/pkg/encoding/plain.go b/pkg/encoding/plain.go
index 388ee8c..50ecb10 100644
--- a/pkg/encoding/plain.go
+++ b/pkg/encoding/plain.go
@@ -42,8 +42,8 @@ var (
)
type plainEncoderPoolDelegator struct {
- name string
pool *sync.Pool
+ name string
size int
}
@@ -71,8 +71,8 @@ func (b *plainEncoderPoolDelegator) Put(encoder SeriesEncoder) {
}
type plainDecoderPoolDelegator struct {
- name string
pool *sync.Pool
+ name string
size int
}
@@ -105,15 +105,15 @@ var (
_ SeriesDecoder = (*plainDecoder)(nil)
)
-// plainEncoder backport to reduced value
+// plainEncoder backport to reduced value.
type plainEncoder struct {
- name string
tsBuff *buffer.Writer
valBuff *buffer.Writer
- len uint32
- num uint32
+ name string
startTime uint64
valueSize int
+ len uint32
+ num uint32
}
func newPlainEncoder() interface{} {
@@ -181,13 +181,13 @@ func (t *plainEncoder) StartTime() uint64 {
}
const (
- // TsLen equals ts(uint64) + data_offset(uint32)
- TsLen = 8 + 4
+ // TSLen equals ts(uint64) + data_offset(uint32).
+ TSLen = 8 + 4
)
var ErrInvalidValue = errors.New("invalid encoded value")
-// plainDecoder decodes encoded time index
+// plainDecoder decodes encoded time index.
type plainDecoder struct {
name string
ts []byte
@@ -264,7 +264,7 @@ func getVal(buf []byte, offset uint32) ([]byte, error) {
}
func getTSSlot(data []byte, index int) []byte {
- return data[index*TsLen : (index+1)*TsLen]
+ return data[index*TSLen : (index+1)*TSLen]
}
func parseTS(tsSlot []byte) uint64 {
diff --git a/pkg/encoding/xor.go b/pkg/encoding/xor.go
index 43203ca..776722b 100644
--- a/pkg/encoding/xor.go
+++ b/pkg/encoding/xor.go
@@ -39,7 +39,7 @@ type XOREncoder struct {
first bool
}
-// NewXOREncoder creates xor zstdEncoder for compressing uint64 data
+// NewXOREncoder creates xor zstdEncoder for compressing uint64 data.
func NewXOREncoder(bw *bit.Writer) *XOREncoder {
return &XOREncoder{
bw: bw,
@@ -82,20 +82,17 @@ func (e *XOREncoder) Write(val uint64) {
}
}
-// XORDecoder decodes buffer to uint64 values using xor compress
+// XORDecoder decodes buffer to uint64 values using xor compress.
type XORDecoder struct {
- val uint64
-
- br *bit.Reader
-
+ err error
+ br *bit.Reader
+ val uint64
leading uint64
trailing uint64
-
- first bool
- err error
+ first bool
}
-// NewXORDecoder create zstdDecoder decompress buffer using xor
+// NewXORDecoder create zstdDecoder decompress buffer using xor.
func NewXORDecoder(br *bit.Reader) *XORDecoder {
s := &XORDecoder{
br: br,
@@ -104,7 +101,7 @@ func NewXORDecoder(br *bit.Reader) *XORDecoder {
return s
}
-// Reset resets the underlying buffer to decode
+// Reset resets the underlying buffer to decode.
func (d *XORDecoder) Reset() {
d.first = true
d.leading = 0
@@ -113,7 +110,7 @@ func (d *XORDecoder) Reset() {
}
// Next return if zstdDecoder has value in buffer using xor, do uncompress logic in next method,
-// data format reference zstdEncoder format
+// data format reference zstdEncoder format.
func (d *XORDecoder) Next() bool {
if d.first {
// read first value
@@ -166,12 +163,12 @@ func (d *XORDecoder) Next() bool {
return true
}
-// Value returns uint64 from buffer
+// Value returns uint64 from buffer.
func (d *XORDecoder) Value() uint64 {
return d.val
}
-// Err returns error raised in Next()
+// Err returns error raised in Next().
func (d *XORDecoder) Err() error {
return d.err
}
diff --git a/pkg/flow/dedup_priority_queue.go b/pkg/flow/dedup_priority_queue.go
index a5c0f8e..86f9c91 100644
--- a/pkg/flow/dedup_priority_queue.go
+++ b/pkg/flow/dedup_priority_queue.go
@@ -32,11 +32,11 @@ type Element interface {
}
// DedupPriorityQueue implements heap.Interface.
-// DedupPriorityQueue is not thread-safe
+// DedupPriorityQueue is not thread-safe.
type DedupPriorityQueue struct {
comparator utils.Comparator
- Items []Element
cache map[Element]struct{}
+ Items []Element
allowDuplicates bool
}
diff --git a/pkg/flow/op.go b/pkg/flow/op.go
index ebd9ca1..e3914a8 100644
--- a/pkg/flow/op.go
+++ b/pkg/flow/op.go
@@ -21,20 +21,20 @@ import (
"context"
)
-// UnaryOperation represents user-defined unary function (i.e. Map, Filter, etc)
+// UnaryOperation represents user-defined unary function (i.e. Map, Filter, etc).
type UnaryOperation[R any] interface {
Apply(ctx context.Context, data interface{}) R
}
-// UnaryFunc implements UnaryOperation as type func (context.Context, interface{})
+// UnaryFunc implements UnaryOperation as type func (context.Context, interface{}).
type UnaryFunc[R any] func(context.Context, interface{}) R
-// Apply implements UnOperation.Apply method
+// Apply implements UnOperation.Apply method.
func (f UnaryFunc[R]) Apply(ctx context.Context, data interface{}) R {
return f(ctx, data)
}
-// FilterFunc transform a function to an UnaryOperation
+// FilterFunc transform a function to an UnaryOperation.
func FilterFunc(filter UnaryOperation[bool]) (UnaryOperation[any], error) {
return UnaryFunc[any](func(ctx context.Context, payload interface{}) interface{} {
predicate := filter.Apply(ctx, payload)
diff --git a/pkg/flow/streaming/flow.go b/pkg/flow/streaming/flow.go
index c6c7a82..8bb6251 100644
--- a/pkg/flow/streaming/flow.go
+++ b/pkg/flow/streaming/flow.go
@@ -32,8 +32,8 @@ type streamingFlow struct {
ctx context.Context
source flow.Source
sink flow.Sink
- ops []flow.Operator
drain chan error
+ ops []flow.Operator
}
func New(source flow.Source) flow.Flow {
diff --git a/pkg/flow/streaming/flow_sliding_window.go b/pkg/flow/streaming/flow_sliding_window.go
index fb9f9c8..853ce55 100644
--- a/pkg/flow/streaming/flow_sliding_window.go
+++ b/pkg/flow/streaming/flow_sliding_window.go
@@ -71,30 +71,17 @@ func (s *windowedFlow) AllowedMaxWindows(windowCnt int) flow.WindowedFlow {
}
type TumblingTimeWindows struct {
- // internal state of the sliding time window
+ errorHandler func(error)
+ snapshots *lru.Cache
+ timerHeap *flow.DedupPriorityQueue
+ aggregationFactory flow.AggregationOpFactory
+ in chan flow.StreamRecord
+ out chan flow.StreamRecord
flow.ComponentState
- // errorHandler is the error handler and set by the streamingFlow
- errorHandler func(error)
- // For TumblingTimeWindows
- // Unit: Milliseconds
- windowSize int64
- // windowCount is the maximum allowed windows kept in the memory
- windowCount int
-
- // thread-safe snapshots
- snapshots *lru.Cache
-
+ windowSize int64
+ windowCount int
currentWatermark int64
- // guard timerHeap
- timerMu sync.Mutex
- timerHeap *flow.DedupPriorityQueue
-
- // aggregationFactory is the factory for creating aggregation operator
- aggregationFactory flow.AggregationOpFactory
-
- // For api.Operator
- in chan flow.StreamRecord
- out chan flow.StreamRecord
+ timerMu sync.Mutex
}
func (s *TumblingTimeWindows) In() chan<- flow.StreamRecord {
@@ -223,7 +210,7 @@ func (s *TumblingTimeWindows) receive() {
// it meets all the following conditions,
// 1) the max timestamp is before the current watermark
// 2) the LRU cache is full
-// 3) the LRU cache does not contain the window entry
+// 3) the LRU cache does not contain the window entry.
func (s *TumblingTimeWindows) isWindowLate(w flow.Window) bool {
return w.MaxTimestamp() <= s.currentWatermark && s.snapshots.Len() >= s.windowCount && !s.snapshots.Contains(w)
}
@@ -259,7 +246,7 @@ func (t timeWindow) MaxTimestamp() int64 {
return t.end - 1
}
-// AssignWindows assigns windows according to the given timestamp
+// AssignWindows assigns windows according to the given timestamp.
func (s *TumblingTimeWindows) AssignWindows(timestamp int64) ([]flow.Window, error) {
if timestamp > math.MinInt64 {
start := getWindowStart(timestamp, s.windowSize)
@@ -279,7 +266,7 @@ func getWindowStart(timestamp, windowSize int64) int64 {
return timestamp - remainder
}
-// eventTimeTriggerOnElement processes element(s) with EventTimeTrigger
+// eventTimeTriggerOnElement processes element(s) with EventTimeTrigger.
func eventTimeTriggerOnElement(window timeWindow, ctx *triggerContext) TriggerResult {
if window.MaxTimestamp() <= ctx.GetCurrentWatermark() {
// if watermark is already past the window fire immediately
@@ -290,8 +277,8 @@ func eventTimeTriggerOnElement(window timeWindow, ctx *triggerContext) TriggerRe
}
type triggerContext struct {
- window timeWindow
delegation *TumblingTimeWindows
+ window timeWindow
}
func (ctx *triggerContext) GetCurrentWatermark() int64 {
diff --git a/pkg/flow/streaming/flow_sliding_window_test.go b/pkg/flow/streaming/flow_sliding_window_test.go
index 798937d..8672527 100644
--- a/pkg/flow/streaming/flow_sliding_window_test.go
+++ b/pkg/flow/streaming/flow_sliding_window_test.go
@@ -56,7 +56,7 @@ func (i *intSumAggregator) Dirty() bool {
var _ = Describe("Sliding Window", func() {
var (
- baseTs time.Time
+ baseTS time.Time
snk *sink.Slice
input []flow.StreamRecord
slidingWindows *TumblingTimeWindows
@@ -67,7 +67,7 @@ var _ = Describe("Sliding Window", func() {
)
BeforeEach(func() {
- baseTs = time.Now()
+ baseTS = time.Now()
})
JustBeforeEach(func() {
@@ -93,7 +93,7 @@ var _ = Describe("Sliding Window", func() {
When("input a single element", func() {
BeforeEach(func() {
input = []flow.StreamRecord{
- flow.NewStreamRecord(1, baseTs.UnixMilli()),
+ flow.NewStreamRecord(1, baseTS.UnixMilli()),
}
})
@@ -106,10 +106,10 @@ var _ = Describe("Sliding Window", func() {
When("input two elements within the same bucket", func() {
BeforeEach(func() {
- baseTs = time.Unix(baseTs.Unix()-baseTs.Unix()%15, 0)
+ baseTS = time.Unix(baseTS.Unix()-baseTS.Unix()%15, 0)
input = []flow.StreamRecord{
- flow.NewStreamRecord(1, baseTs.UnixMilli()),
- flow.NewStreamRecord(2, baseTs.Add(time.Second*5).UnixMilli()),
+ flow.NewStreamRecord(1, baseTS.UnixMilli()),
+ flow.NewStreamRecord(2, baseTS.Add(time.Second*5).UnixMilli()),
}
})
@@ -122,10 +122,10 @@ var _ = Describe("Sliding Window", func() {
When("input two elements within adjacent buckets", func() {
BeforeEach(func() {
- baseTs = time.Unix(baseTs.Unix()-baseTs.Unix()%15+14, 0)
+ baseTS = time.Unix(baseTS.Unix()-baseTS.Unix()%15+14, 0)
input = []flow.StreamRecord{
- flow.NewStreamRecord(1, baseTs.UnixMilli()),
- flow.NewStreamRecord(2, baseTs.Add(time.Second*5).UnixMilli()),
+ flow.NewStreamRecord(1, baseTS.UnixMilli()),
+ flow.NewStreamRecord(2, baseTS.Add(time.Second*5).UnixMilli()),
}
})
diff --git a/pkg/flow/streaming/flow_test.go b/pkg/flow/streaming/flow_test.go
index 989571f..5ca82dd 100644
--- a/pkg/flow/streaming/flow_test.go
+++ b/pkg/flow/streaming/flow_test.go
@@ -31,7 +31,7 @@ import (
)
// numberRange generates a slice with `count` number of integers starting from `begin`,
-// i.e. [begin, begin + count)
+// i.e. [begin, begin + count).
func numberRange(begin, count int) []int {
result := make([]int, 0)
for i := 0; i < count; i++ {
diff --git a/pkg/flow/streaming/flow_topn.go b/pkg/flow/streaming/flow_topn.go
index 2a52e21..7be2fa4 100644
--- a/pkg/flow/streaming/flow_topn.go
+++ b/pkg/flow/streaming/flow_topn.go
@@ -68,19 +68,13 @@ func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
}
type topNAggregator struct {
- // cacheSize is the maximum number of entries which can be held in the buffer, i.e. treeMap
- cacheSize int
- // currentTopNum indicates how many records are tracked.
- // This should not exceed cacheSize
- currentTopNum int
- treeMap *treemap.Map
- // sortKeyExtractor is an extractor to fetch sort key from the record
- // TODO: currently we only support sorting numeric field, i.e. int64
+ treeMap *treemap.Map
sortKeyExtractor func(flow.StreamRecord) int64
- // sort indicates the order of results
- sort TopNSort
- comparator utils.Comparator
- dirty bool
+ comparator utils.Comparator
+ cacheSize int
+ currentTopNum int
+ sort TopNSort
+ dirty bool
}
type TopNOption func(aggregator *topNAggregator)
diff --git a/pkg/flow/streaming/flow_topn_test.go b/pkg/flow/streaming/flow_topn_test.go
index a00f090..bc2592e 100644
--- a/pkg/flow/streaming/flow_topn_test.go
+++ b/pkg/flow/streaming/flow_topn_test.go
@@ -43,8 +43,8 @@ func TestFlow_TopN_Aggregator(t *testing.T) {
}
tests := []struct {
name string
- sort TopNSort
expected []*Tuple2
+ sort TopNSort
}{
{
name: "DESC",
diff --git a/pkg/flow/streaming/flow_unary.go b/pkg/flow/streaming/flow_unary.go
index f12f2f8..22a8ede 100644
--- a/pkg/flow/streaming/flow_unary.go
+++ b/pkg/flow/streaming/flow_unary.go
@@ -46,16 +46,16 @@ func (f *streamingFlow) Transform(op flow.UnaryOperation[any]) flow.Flow {
var _ flow.Operator = (*unaryOperator)(nil)
type unaryOperator struct {
+ op flow.UnaryOperation[any]
+ in chan flow.StreamRecord
+ out chan flow.StreamRecord
flow.ComponentState
- op flow.UnaryOperation[any]
- in chan flow.StreamRecord
- out chan flow.StreamRecord
parallelism uint
}
func (u *unaryOperator) Setup(ctx context.Context) error {
// run a background job as a consumer
- go u.run()
+ go u.run(ctx)
return nil
}
@@ -87,13 +87,13 @@ func (u *unaryOperator) Out() <-chan flow.StreamRecord {
return u.out
}
-func (u *unaryOperator) run() {
+func (u *unaryOperator) run(ctx context.Context) {
semaphore := make(chan struct{}, u.parallelism)
for elem := range u.in {
semaphore <- struct{}{}
go func(r flow.StreamRecord) {
defer func() { <-semaphore }()
- result := u.op.Apply(context.TODO(), r.Data())
+ result := u.op.Apply(ctx, r.Data())
switch val := result.(type) {
case nil:
return
diff --git a/pkg/flow/streaming/sink/slice.go b/pkg/flow/streaming/sink/slice.go
index ad55850..b755aa5 100644
--- a/pkg/flow/streaming/sink/slice.go
+++ b/pkg/flow/streaming/sink/slice.go
@@ -27,10 +27,10 @@ import (
var _ flow.Sink = (*Slice)(nil)
type Slice struct {
- sync.RWMutex
- flow.ComponentState
- slice []interface{}
in chan flow.StreamRecord
+ slice []interface{}
+ flow.ComponentState
+ sync.RWMutex
}
func NewSlice() *Slice {
diff --git a/pkg/flow/streaming/sources/channel.go b/pkg/flow/streaming/sources/channel.go
index eb71354..5ab7c7b 100644
--- a/pkg/flow/streaming/sources/channel.go
+++ b/pkg/flow/streaming/sources/channel.go
@@ -29,9 +29,9 @@ import (
var _ flow.Source = (*sourceChan)(nil)
type sourceChan struct {
- flow.ComponentState
in interface{}
out chan flow.StreamRecord
+ flow.ComponentState
}
func (s *sourceChan) Out() <-chan flow.StreamRecord {
diff --git a/pkg/flow/types.go b/pkg/flow/types.go
index 9efaefb..5b61910 100644
--- a/pkg/flow/types.go
+++ b/pkg/flow/types.go
@@ -28,7 +28,7 @@ import (
type Data []any
// Flow is an abstraction of data flow for
-// both Streaming and Batch
+// both Streaming and Batch.
type Flow interface {
io.Closer
// Filter is used to filter data.
@@ -87,8 +87,8 @@ type AggregationOpFactory func() AggregationOp
// StreamRecord is a container wraps user data and timestamp.
// It is the underlying transmission medium for the streaming processing.
type StreamRecord struct {
- ts int64
data interface{}
+ ts int64
}
func NewStreamRecord(data interface{}, ts int64) StreamRecord {
diff --git a/pkg/grpchelper/client.go b/pkg/grpchelper/client.go
index 0c95ecf..f482ebd 100644
--- a/pkg/grpchelper/client.go
+++ b/pkg/grpchelper/client.go
@@ -18,6 +18,7 @@ package grpchelper
import (
"context"
+ "errors"
"time"
"google.golang.org/grpc"
@@ -40,7 +41,7 @@ func Conn(addr string, connTimeout time.Duration, opts ...grpc.DialOption) (*grp
defer dialCancel()
conn, err := grpc.DialContext(dialCtx, addr, opts...)
if err != nil {
- if err == context.DeadlineExceeded {
+ if errors.Is(err, context.DeadlineExceeded) {
l.Warn().Str("addr", addr).Dur("timeout", connTimeout).Msg("timeout: failed to connect service")
} else {
l.Warn().Str("addr", addr).Err(err).Msg("error: failed to connect service")
diff --git a/pkg/index/index.go b/pkg/index/index.go
index 44051fe..03d28ad 100644
--- a/pkg/index/index.go
+++ b/pkg/index/index.go
@@ -66,8 +66,8 @@ func (f FieldKey) Equal(other FieldKey) bool {
}
type Field struct {
- Key FieldKey
Term []byte
+ Key FieldKey
}
func (f Field) Marshal() ([]byte, error) {
@@ -145,8 +145,8 @@ func (i *emptyIterator) Close() error {
}
type PostingValue struct {
- Term []byte
Value posting.List
+ Term []byte
}
type Writer interface {
diff --git a/pkg/index/inverted/inverted.go b/pkg/index/inverted/inverted.go
index 0797a8f..a77ab73 100644
--- a/pkg/index/inverted/inverted.go
+++ b/pkg/index/inverted/inverted.go
@@ -62,8 +62,8 @@ func init() {
var _ index.Store = (*store)(nil)
type StoreOpts struct {
- Path string
Logger *logger.Logger
+ Path string
}
type doc struct {
@@ -306,13 +306,11 @@ func (s *store) flush() {
type blugeMatchIterator struct {
delegated search.DocumentMatchIterator
+ err error
+ current *index.PostingValue
+ agg *index.PostingValue
fieldKey string
-
- current *index.PostingValue
- agg *index.PostingValue
-
- closed bool
- err error
+ closed bool
}
func newBlugeMatchIterator(delegated search.DocumentMatchIterator, fieldKey string) blugeMatchIterator {
diff --git a/pkg/index/inverted/inverted_test.go b/pkg/index/inverted/inverted_test.go
index ebcdce1..a1109f9 100644
--- a/pkg/index/inverted/inverted_test.go
+++ b/pkg/index/inverted/inverted_test.go
@@ -90,8 +90,8 @@ func TestStore_Match(t *testing.T) {
s.(*store).flush()
tests := []struct {
- matches []string
want posting.List
+ matches []string
wantErr bool
}{
{
diff --git a/pkg/index/iterator.go b/pkg/index/iterator.go
index 5a350a2..0f51300 100644
--- a/pkg/index/iterator.go
+++ b/pkg/index/iterator.go
@@ -38,15 +38,14 @@ var (
)
type FieldIteratorTemplate struct {
+ err error
delegated *delegateIterator
-
- init bool
cur *PostingValue
- err error
- termRange RangeOpts
fn CompositePostingValueFn
- reverse bool
seekKey []byte
+ termRange RangeOpts
+ init bool
+ reverse bool
}
func (f *FieldIteratorTemplate) Next() bool {
@@ -146,11 +145,11 @@ type SwitchFn = func(a, b []byte) bool
var _ FieldIterator = (*mergedIterator)(nil)
type mergedIterator struct {
+ cur *PostingValue
+ switchFn SwitchFn
inner []FieldIterator
drained []FieldIterator
drainedCount int
- cur *PostingValue
- switchFn SwitchFn
init bool
closed bool
}
@@ -232,12 +231,11 @@ var _ kv.Iterator = (*delegateIterator)(nil)
type delegateIterator struct {
delegated kv.Iterator
- fieldKey FieldKey
- fieldKeyBytes []byte
l *logger.Logger
-
- curField Field
- closed bool
+ fieldKeyBytes []byte
+ curField Field
+ fieldKey FieldKey
+ closed bool
}
func newDelegateIterator(delegated kv.Iterator, fieldKey FieldKey, l *logger.Logger) *delegateIterator {
diff --git a/pkg/index/lsm/lsm.go b/pkg/index/lsm/lsm.go
index 0a26c26..dff5eaf 100644
--- a/pkg/index/lsm/lsm.go
+++ b/pkg/index/lsm/lsm.go
@@ -57,8 +57,8 @@ func (s *store) Write(fields []index.Field, itemID common.ItemID) (err error) {
}
type StoreOpts struct {
- Path string
Logger *logger.Logger
+ Path string
MemTableSize int64
}
diff --git a/pkg/index/testcases/duration.go b/pkg/index/testcases/duration.go
index df79010..8fd998d 100644
--- a/pkg/index/testcases/duration.go
+++ b/pkg/index/testcases/duration.go
@@ -44,14 +44,14 @@ type SimpleStore interface {
}
type args struct {
- fieldKey index.FieldKey
termRange index.RangeOpts
+ fieldKey index.FieldKey
orderType modelv1.Sort
}
type result struct {
- key int
items []int
+ key int
}
func RunDuration(t *testing.T, data map[int]posting.List, store SimpleStore) {
@@ -59,8 +59,8 @@ func RunDuration(t *testing.T, data map[int]posting.List, store SimpleStore) {
is := require.New(t)
tests := []struct {
name string
- args args
want []int
+ args args
}{
{
name: "sort in asc order",
diff --git a/pkg/index/testcases/service_name.go b/pkg/index/testcases/service_name.go
index 60d7e18..9b024a0 100644
--- a/pkg/index/testcases/service_name.go
+++ b/pkg/index/testcases/service_name.go
@@ -36,9 +36,9 @@ var serviceName = index.FieldKey{
func RunServiceName(t *testing.T, store SimpleStore) {
tester := assert.New(t)
tests := []struct {
+ want posting.List
name string
arg index.Field
- want posting.List
wantErr bool
}{
{
diff --git a/pkg/logger/jsonproto.go b/pkg/logger/jsonproto.go
index 9c87037..c1f8137 100644
--- a/pkg/logger/jsonproto.go
+++ b/pkg/logger/jsonproto.go
@@ -31,7 +31,11 @@ type MarshalError struct {
func Proto(message proto.Message) []byte {
b, err := protojson.Marshal(message)
if err != nil {
- b, _ = json.Marshal(MarshalError{Msg: err.Error()})
+ var errJSON error
+ b, errJSON = json.Marshal(MarshalError{Msg: err.Error()})
+ if errJSON != nil {
+ return nil
+ }
}
return b
}
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index cd37bcd..2b41ff7 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -32,7 +32,7 @@ var (
type contextKey struct{}
-// Logging is the config info
+// Logging is the config info.
type Logging struct {
Env string
Level string
@@ -42,9 +42,9 @@ type Logging struct {
// Logger is wrapper for rs/zerolog logger with module, it is singleton.
type Logger struct {
- module string
*zerolog.Logger
modules map[string]zerolog.Level
+ module string
}
func (l Logger) Module() string {
@@ -75,7 +75,7 @@ func (l *Logger) Named(name ...string) *Logger {
return &Logger{module: module, modules: l.modules, Logger: &subLogger}
}
-// Loggable indicates the implement supports logging
+// Loggable indicates the implement supports logging.
type Loggable interface {
SetLogger(*Logger)
}
diff --git a/pkg/logger/setting.go b/pkg/logger/setting.go
index ebf923c..1018bfe 100644
--- a/pkg/logger/setting.go
+++ b/pkg/logger/setting.go
@@ -35,9 +35,9 @@ const rootName = "ROOT"
var root = rootLogger{}
type rootLogger struct {
- done uint32
- m sync.Mutex
l *Logger
+ m sync.Mutex
+ done uint32
}
func (rl *rootLogger) verify() {
@@ -74,7 +74,7 @@ func (rl *rootLogger) set(cfg Logging) error {
return nil
}
-// GetLogger return logger with a scope
+// GetLogger return logger with a scope.
func GetLogger(scope ...string) *Logger {
root.verify()
if len(scope) < 1 {
@@ -87,12 +87,12 @@ func GetLogger(scope ...string) *Logger {
return l
}
-// Init initializes a rs/zerolog logger from user config
+// Init initializes a rs/zerolog logger from user config.
func Init(cfg Logging) (err error) {
return root.set(cfg)
}
-// getLogger initializes a root logger
+// getLogger initializes a root logger.
func getLogger(cfg Logging) (*Logger, error) {
modules := make(map[string]zerolog.Level)
if len(cfg.Modules) > 0 {
diff --git a/pkg/pb/v1/metadata.go b/pkg/pb/v1/metadata.go
index b5eef8c..13b3202 100644
--- a/pkg/pb/v1/metadata.go
+++ b/pkg/pb/v1/metadata.go
@@ -18,11 +18,11 @@
package v1
import (
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
-func FindTagByName(families []*database_v1.TagFamilySpec, tagName string) (int, int, *database_v1.TagSpec) {
+func FindTagByName(families []*databasev1.TagFamilySpec, tagName string) (int, int, *databasev1.TagSpec) {
for fi, family := range families {
for ti, tag := range family.Tags {
if tagName == tag.GetName() {
@@ -33,41 +33,41 @@ func FindTagByName(families []*database_v1.TagFamilySpec, tagName string) (int,
return 0, 0, nil
}
-func TagValueTypeConv(tagValue *model_v1.TagValue) (tagType database_v1.TagType, isNull bool) {
+func TagValueTypeConv(tagValue *modelv1.TagValue) (tagType databasev1.TagType, isNull bool) {
switch tagValue.GetValue().(type) {
- case *model_v1.TagValue_Int:
- return database_v1.TagType_TAG_TYPE_INT, false
- case *model_v1.TagValue_Str:
- return database_v1.TagType_TAG_TYPE_STRING, false
- case *model_v1.TagValue_IntArray:
- return database_v1.TagType_TAG_TYPE_INT_ARRAY, false
- case *model_v1.TagValue_StrArray:
- return database_v1.TagType_TAG_TYPE_STRING_ARRAY, false
- case *model_v1.TagValue_BinaryData:
- return database_v1.TagType_TAG_TYPE_DATA_BINARY, false
- case *model_v1.TagValue_Id:
- return database_v1.TagType_TAG_TYPE_ID, false
- case *model_v1.TagValue_Null:
- return database_v1.TagType_TAG_TYPE_UNSPECIFIED, true
+ case *modelv1.TagValue_Int:
+ return databasev1.TagType_TAG_TYPE_INT, false
+ case *modelv1.TagValue_Str:
+ return databasev1.TagType_TAG_TYPE_STRING, false
+ case *modelv1.TagValue_IntArray:
+ return databasev1.TagType_TAG_TYPE_INT_ARRAY, false
+ case *modelv1.TagValue_StrArray:
+ return databasev1.TagType_TAG_TYPE_STRING_ARRAY, false
+ case *modelv1.TagValue_BinaryData:
+ return databasev1.TagType_TAG_TYPE_DATA_BINARY, false
+ case *modelv1.TagValue_Id:
+ return databasev1.TagType_TAG_TYPE_ID, false
+ case *modelv1.TagValue_Null:
+ return databasev1.TagType_TAG_TYPE_UNSPECIFIED, true
}
- return database_v1.TagType_TAG_TYPE_UNSPECIFIED, false
+ return databasev1.TagType_TAG_TYPE_UNSPECIFIED, false
}
-func FieldValueTypeConv(tagValue *model_v1.FieldValue) (tagType database_v1.FieldType, isNull bool) {
+func FieldValueTypeConv(tagValue *modelv1.FieldValue) (tagType databasev1.FieldType, isNull bool) {
switch tagValue.GetValue().(type) {
- case *model_v1.FieldValue_Int:
- return database_v1.FieldType_FIELD_TYPE_INT, false
- case *model_v1.FieldValue_Str:
- return database_v1.FieldType_FIELD_TYPE_STRING, false
- case *model_v1.FieldValue_BinaryData:
- return database_v1.FieldType_FIELD_TYPE_DATA_BINARY, false
- case *model_v1.FieldValue_Null:
- return database_v1.FieldType_FIELD_TYPE_UNSPECIFIED, true
+ case *modelv1.FieldValue_Int:
+ return databasev1.FieldType_FIELD_TYPE_INT, false
+ case *modelv1.FieldValue_Str:
+ return databasev1.FieldType_FIELD_TYPE_STRING, false
+ case *modelv1.FieldValue_BinaryData:
+ return databasev1.FieldType_FIELD_TYPE_DATA_BINARY, false
+ case *modelv1.FieldValue_Null:
+ return databasev1.FieldType_FIELD_TYPE_UNSPECIFIED, true
}
- return database_v1.FieldType_FIELD_TYPE_UNSPECIFIED, false
+ return databasev1.FieldType_FIELD_TYPE_UNSPECIFIED, false
}
-func ParseMaxModRevision(indexRules []*database_v1.IndexRule) (maxRevisionForIdxRules int64) {
+func ParseMaxModRevision(indexRules []*databasev1.IndexRule) (maxRevisionForIdxRules int64) {
maxRevisionForIdxRules = int64(0)
for _, idxRule := range indexRules {
if idxRule.GetMetadata().GetModRevision() > maxRevisionForIdxRules {
diff --git a/pkg/pb/v1/tsdb/interval.go b/pkg/pb/v1/tsdb/interval.go
index ace265d..0019645 100644
--- a/pkg/pb/v1/tsdb/interval.go
+++ b/pkg/pb/v1/tsdb/interval.go
@@ -20,17 +20,17 @@ package tsdb
import (
"errors"
- common_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
)
var ErrInvalidUnit = errors.New("invalid interval rule's unit")
-func ToIntervalRule(ir *common_v1.IntervalRule) (result tsdb.IntervalRule, err error) {
+func ToIntervalRule(ir *commonv1.IntervalRule) (result tsdb.IntervalRule, err error) {
switch ir.Unit {
- case common_v1.IntervalRule_UNIT_DAY:
+ case commonv1.IntervalRule_UNIT_DAY:
result.Unit = tsdb.DAY
- case common_v1.IntervalRule_UNIT_HOUR:
+ case commonv1.IntervalRule_UNIT_HOUR:
result.Unit = tsdb.HOUR
default:
return result, ErrInvalidUnit
diff --git a/pkg/query/logical/common.go b/pkg/query/logical/common.go
index f5f634e..92ceef7 100644
--- a/pkg/query/logical/common.go
+++ b/pkg/query/logical/common.go
@@ -157,7 +157,7 @@ func NewTag(family, name string) *Tag {
}
}
-// NewTags create an array of Tag within a TagFamily
+// NewTags create an array of Tag within a TagFamily.
func NewTags(family string, tagNames ...string) []*Tag {
tags := make([]*Tag, len(tagNames))
for i, name := range tagNames {
@@ -166,7 +166,7 @@ func NewTags(family string, tagNames ...string) []*Tag {
return tags
}
-// GetCompoundName is only used for error message
+// GetCompoundName is only used for error message.
func (t *Tag) GetCompoundName() string {
return t.familyName + ":" + t.name
}
diff --git a/pkg/query/logical/expr.go b/pkg/query/logical/expr.go
index 3664d28..0775855 100644
--- a/pkg/query/logical/expr.go
+++ b/pkg/query/logical/expr.go
@@ -26,7 +26,7 @@ import (
var _ ResolvableExpr = (*TagRef)(nil)
// TagRef is the reference to the field
-// also it holds the definition (derived from the streamSchema, measureSchema) of the tag
+// also it holds the definition (derived from the streamSchema, measureSchema) of the tag.
type TagRef struct {
// Tag defines the family name and name of the Tag
Tag *Tag
@@ -67,7 +67,7 @@ func NewTagRef(familyName, tagName string) *TagRef {
}
}
-// NewSearchableTagRef is a short-handed method for creating a TagRef to the tag in the searchable family
+// NewSearchableTagRef is a short-handed method for creating a TagRef to the tag in the searchable family.
func NewSearchableTagRef(tagName string) *TagRef {
return &TagRef{
Tag: NewTag("searchable", tagName),
@@ -77,7 +77,7 @@ func NewSearchableTagRef(tagName string) *TagRef {
var _ ResolvableExpr = (*FieldRef)(nil)
// FieldRef is the reference to the field
-// also it holds the definition (derived from measureSchema) of the field
+// also it holds the definition (derived from measureSchema) of the field.
type FieldRef struct {
// Field defines the name of the Field
Field *Field
@@ -113,6 +113,6 @@ func (f *FieldRef) Resolve(s Schema) error {
}
type FieldSpec struct {
- FieldIdx int
Spec *databasev1.FieldSpec
+ FieldIdx int
}
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index a0194b3..604161b 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -26,8 +26,8 @@ import (
"github.com/pkg/errors"
"github.com/apache/skywalking-banyandb/api/common"
- database_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/index"
@@ -41,18 +41,18 @@ var (
)
type GlobalIndexError struct {
- IndexRule *database_v1.IndexRule
+ IndexRule *databasev1.IndexRule
Expr LiteralExpr
}
-func (g *GlobalIndexError) Error() string { return g.IndexRule.String() }
+func (g GlobalIndexError) Error() string { return g.IndexRule.String() }
-func BuildLocalFilter(criteria *model_v1.Criteria, schema Schema, entityDict map[string]int, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
+func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[string]int, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
if criteria == nil {
return nil, []tsdb.Entity{entity}, nil
}
switch criteria.GetExp().(type) {
- case *model_v1.Criteria_Condition:
+ case *modelv1.Criteria_Condition:
cond := criteria.GetCondition()
expr, parsedEntity, err := parseExprOrEntity(entityDict, entity, cond)
if err != nil {
@@ -62,7 +62,7 @@ func BuildLocalFilter(criteria *model_v1.Criteria, schema Schema, entityDict map
return nil, []tsdb.Entity{parsedEntity}, nil
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
- if indexRule.Location == database_v1.IndexRule_LOCATION_GLOBAL {
+ if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
return nil, nil, &GlobalIndexError{
IndexRule: indexRule,
Expr: expr,
@@ -71,7 +71,7 @@ func BuildLocalFilter(criteria *model_v1.Criteria, schema Schema, entityDict map
return parseCondition(cond, indexRule, expr, entity)
}
return eNode, []tsdb.Entity{entity}, nil
- case *model_v1.Criteria_Le:
+ case *modelv1.Criteria_Le:
le := criteria.GetLe()
if le.GetLeft() == nil && le.GetRight() == nil {
return nil, nil, errors.WithMessagef(ErrInvalidLogicalExpression, "both sides(left and right) of [%v] are empty", criteria)
@@ -92,90 +92,93 @@ func BuildLocalFilter(criteria *model_v1.Criteria, schema Schema, entityDict map
return nil, entities, nil
}
switch le.Op {
- case model_v1.LogicalExpression_LOGICAL_OP_AND:
+ case modelv1.LogicalExpression_LOGICAL_OP_AND:
and := newAnd(2)
and.append(left).append(right)
return and, entities, nil
- case model_v1.LogicalExpression_LOGICAL_OP_OR:
+ case modelv1.LogicalExpression_LOGICAL_OP_OR:
or := newOr(2)
or.append(left).append(right)
return or, entities, nil
}
-
}
return nil, nil, ErrInvalidCriteriaType
}
-func parseCondition(cond *model_v1.Condition, indexRule *database_v1.IndexRule, expr LiteralExpr, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
+func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, expr LiteralExpr, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
switch cond.Op {
- case model_v1.Condition_BINARY_OP_GT:
+ case modelv1.Condition_BINARY_OP_GT:
return newRange(indexRule, index.RangeOpts{
Lower: bytes.Join(expr.Bytes(), nil),
}), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_GE:
+ case modelv1.Condition_BINARY_OP_GE:
return newRange(indexRule, index.RangeOpts{
IncludesLower: true,
Lower: bytes.Join(expr.Bytes(), nil),
}), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_LT:
+ case modelv1.Condition_BINARY_OP_LT:
return newRange(indexRule, index.RangeOpts{
Upper: bytes.Join(expr.Bytes(), nil),
}), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_LE:
+ case modelv1.Condition_BINARY_OP_LE:
return newRange(indexRule, index.RangeOpts{
IncludesUpper: true,
Upper: bytes.Join(expr.Bytes(), nil),
}), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_EQ:
+ case modelv1.Condition_BINARY_OP_EQ:
return newEq(indexRule, expr), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_MATCH:
+ case modelv1.Condition_BINARY_OP_MATCH:
return newMatch(indexRule, expr), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_NE:
+ case modelv1.Condition_BINARY_OP_NE:
return newNot(indexRule, newEq(indexRule, expr)), []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_HAVING:
+ case modelv1.Condition_BINARY_OP_HAVING:
bb := expr.Bytes()
and := newAnd(len(bb))
for _, b := range bb {
and.append(newEq(indexRule, newBytesLiteral(b)))
}
return and, []tsdb.Entity{entity}, nil
- case model_v1.Condition_BINARY_OP_NOT_HAVING:
+ case modelv1.Condition_BINARY_OP_NOT_HAVING:
bb := expr.Bytes()
and := newAnd(len(bb))
for _, b := range bb {
and.append(newEq(indexRule, newBytesLiteral(b)))
}
return newNot(indexRule, and), []tsdb.Entity{entity}, nil
+ case modelv1.Condition_BINARY_OP_IN:
+ panic("unimplemented")
+ case modelv1.Condition_BINARY_OP_NOT_IN:
+ panic("unimplemented")
}
return nil, nil, errors.WithMessagef(ErrUnsupportedConditionOp, "index filter parses %v", cond)
}
-func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *model_v1.Condition) (LiteralExpr, tsdb.Entity, error) {
+func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, tsdb.Entity, error) {
parsedEntity := make(tsdb.Entity, len(entity))
copy(parsedEntity, entity)
entityIdx, ok := entityDict[cond.Name]
- if ok && cond.Op != model_v1.Condition_BINARY_OP_EQ {
+ if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ {
return nil, nil, errors.WithMessagef(ErrUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
}
switch v := cond.Value.Value.(type) {
- case *model_v1.TagValue_Str:
+ case *modelv1.TagValue_Str:
if ok {
parsedEntity[entityIdx] = []byte(v.Str.GetValue())
return nil, parsedEntity, nil
}
return Str(v.Str.GetValue()), nil, nil
- case *model_v1.TagValue_Id:
+ case *modelv1.TagValue_Id:
if ok {
parsedEntity[entityIdx] = []byte(v.Id.GetValue())
return nil, parsedEntity, nil
}
return ID(v.Id.GetValue()), nil, nil
- case *model_v1.TagValue_StrArray:
+ case *modelv1.TagValue_StrArray:
return &strArrLiteral{
arr: v.StrArray.GetValue(),
}, nil, nil
- case *model_v1.TagValue_Int:
+ case *modelv1.TagValue_Int:
if ok {
parsedEntity[entityIdx] = convert.Int64ToBytes(v.Int.GetValue())
return nil, parsedEntity, nil
@@ -183,24 +186,24 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
return &int64Literal{
int64: v.Int.GetValue(),
}, nil, nil
- case *model_v1.TagValue_IntArray:
+ case *modelv1.TagValue_IntArray:
return &int64ArrLiteral{
arr: v.IntArray.GetValue(),
}, nil, nil
- case *model_v1.TagValue_Null:
+ case *modelv1.TagValue_Null:
return nullLiteralExpr, nil, nil
}
return nil, nil, errors.WithMessagef(ErrUnsupportedConditionValue, "index filter parses %v", cond)
}
-func parseEntities(op model_v1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
+func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
count := len(input)
result := make(tsdb.Entity, count)
mergedEntities := make([]tsdb.Entity, 0, len(left)+len(right))
mergedEntities = append(mergedEntities, left...)
mergedEntities = append(mergedEntities, right...)
switch op {
- case model_v1.LogicalExpression_LOGICAL_OP_AND:
+ case modelv1.LogicalExpression_LOGICAL_OP_AND:
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -216,7 +219,7 @@ func parseEntities(op model_v1.LogicalExpression_LogicalOp, input tsdb.Entity, l
}
result[i] = entry
}
- case model_v1.LogicalExpression_LOGICAL_OP_OR:
+ case modelv1.LogicalExpression_LOGICAL_OP_OR:
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -234,10 +237,10 @@ func parseEntities(op model_v1.LogicalExpression_LogicalOp, input tsdb.Entity, l
}
type FieldKey struct {
- *database_v1.IndexRule
+ *databasev1.IndexRule
}
-func newFieldKey(indexRule *database_v1.IndexRule) FieldKey {
+func newFieldKey(indexRule *databasev1.IndexRule) FieldKey {
return FieldKey{indexRule}
}
@@ -395,7 +398,7 @@ type not struct {
Inner index.Filter
}
-func newNot(indexRule *database_v1.IndexRule, inner index.Filter) *not {
+func newNot(indexRule *databasev1.IndexRule, inner index.Filter) *not {
return ¬{
Key: newFieldKey(indexRule),
Inner: inner,
@@ -433,7 +436,7 @@ type eq struct {
*leaf
}
-func newEq(indexRule *database_v1.IndexRule, values LiteralExpr) *eq {
+func newEq(indexRule *databasev1.IndexRule, values LiteralExpr) *eq {
return &eq{
leaf: &leaf{
Key: newFieldKey(indexRule),
@@ -467,7 +470,7 @@ type match struct {
*leaf
}
-func newMatch(indexRule *database_v1.IndexRule, values LiteralExpr) *match {
+func newMatch(indexRule *databasev1.IndexRule, values LiteralExpr) *match {
return &match{
leaf: &leaf{
Key: newFieldKey(indexRule),
@@ -507,7 +510,7 @@ type rangeOp struct {
Opts index.RangeOpts
}
-func newRange(indexRule *database_v1.IndexRule, opts index.RangeOpts) *rangeOp {
+func newRange(indexRule *databasev1.IndexRule, opts index.RangeOpts) *rangeOp {
return &rangeOp{
leaf: &leaf{
Key: newFieldKey(indexRule),
diff --git a/pkg/query/logical/iter.go b/pkg/query/logical/iter.go
index bc1aae9..29d00fc 100644
--- a/pkg/query/logical/iter.go
+++ b/pkg/query/logical/iter.go
@@ -32,7 +32,7 @@ type ItemIterator interface {
var _ heap.Interface = (*containerHeap)(nil)
-// container contains both iter and its current item
+// container contains both iter and its current item.
type container struct {
c Comparator
item tsdb.Item
@@ -58,16 +58,9 @@ func (h *containerHeap) Pop() interface{} {
}
type itemIter struct {
- // c is the comparator to sort items
- c Comparator
- // iters is the list of initial Iterator
+ c Comparator
+ h *containerHeap
iters []tsdb.Iterator
- // deq is the deque of the container
- // 1. When we push a new container, we can normally append it to the tail of the deq,
- // and then sort the whole slice
- // 2. When we pop a new container, we can just pop out the first element in the deq.
- // The rest of the slice is still sorted.
- h *containerHeap
}
func NewItemIter(iters []tsdb.Iterator, c Comparator) ItemIterator {
diff --git a/pkg/query/logical/measure/measure_analyzer.go b/pkg/query/logical/measure/measure_analyzer.go
index db48e09..312ab87 100644
--- a/pkg/query/logical/measure/measure_analyzer.go
+++ b/pkg/query/logical/measure/measure_analyzer.go
@@ -47,7 +47,7 @@ func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata)
return nil, err
}
- indexRules, err := a.metadataRepoImpl.IndexRules(context.TODO(), metadata)
+ indexRules, err := a.metadataRepoImpl.IndexRules(ctx, metadata)
if err != nil {
return nil, err
}
@@ -127,7 +127,7 @@ func (a *Analyzer) Analyze(_ context.Context, criteria *measurev1.QueryRequest,
// parseFields parses the query request to decide which kind of plan should be generated
// Basically,
// 1 - If no criteria is given, we can only scan all shards
-// 2 - If criteria is given, but all of those fields exist in the "entity" definition
+// 2 - If criteria is given, but all of those fields exist in the "entity" definition.
func parseFields(criteria *measurev1.QueryRequest, metadata *commonv1.Metadata, s logical.Schema, groupByEntity bool) (logical.UnresolvedPlan, error) {
timeRange := criteria.GetTimeRange()
diff --git a/pkg/query/logical/measure/measure_plan_aggregation.go b/pkg/query/logical/measure/measure_plan_aggregation.go
index ddbe355..4760210 100644
--- a/pkg/query/logical/measure/measure_plan_aggregation.go
+++ b/pkg/query/logical/measure/measure_plan_aggregation.go
@@ -35,11 +35,9 @@ var (
)
type unresolvedAggregation struct {
- unresolvedInput logical.UnresolvedPlan
- // aggrFunc is the type of aggregation
- aggrFunc modelv1.AggregationFunction
- // groupBy should be a subset of tag projection
+ unresolvedInput logical.UnresolvedPlan
aggregationField *logical.Field
+ aggrFunc modelv1.AggregationFunction
isGroup bool
}
diff --git a/pkg/query/logical/measure/measure_plan_groupby.go b/pkg/query/logical/measure/measure_plan_groupby.go
index 444da8b..26e5a0b 100644
--- a/pkg/query/logical/measure/measure_plan_groupby.go
+++ b/pkg/query/logical/measure/measure_plan_groupby.go
@@ -202,15 +202,14 @@ func (gmi *groupIterator) Close() error {
}
type groupSortIterator struct {
- groupByTagsRefs [][]*logical.TagRef
iter executor.MIterator
+ err error
+ cdp *measurev1.DataPoint
+ groupByTagsRefs [][]*logical.TagRef
+ current []*measurev1.DataPoint
index int
-
- current []*measurev1.DataPoint
- cdp *measurev1.DataPoint
- key uint64
- closed bool
- err error
+ key uint64
+ closed bool
}
func newGroupSortIterator(iter executor.MIterator, groupByTagsRefs [][]*logical.TagRef) executor.MIterator {
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index 52e1898..96f0fc6 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -41,13 +41,13 @@ var _ logical.UnresolvedPlan = (*unresolvedIndexScan)(nil)
type unresolvedIndexScan struct {
startTime time.Time
endTime time.Time
- metadata *commonv1.Metadata
filter index.Filter
+ metadata *commonv1.Metadata
+ unresolvedOrderBy *logical.UnresolvedOrderBy
projectionTags [][]*logical.Tag
projectionFields []*logical.Field
entities []tsdb.Entity
groupByEntity bool
- unresolvedOrderBy *logical.UnresolvedOrderBy
}
func (uis *unresolvedIndexScan) Analyze(s logical.Schema) (logical.Plan, error) {
@@ -92,16 +92,16 @@ func (uis *unresolvedIndexScan) Analyze(s logical.Schema) (logical.Plan, error)
var _ logical.Plan = (*localIndexScan)(nil)
type localIndexScan struct {
+ schema logical.Schema
+ filter index.Filter
*logical.OrderBy
- timeRange timestamp.TimeRange
- schema logical.Schema
metadata *commonv1.Metadata
- filter index.Filter
+ l *logger.Logger
+ timeRange timestamp.TimeRange
projectionTagsRefs [][]*logical.TagRef
projectionFieldsRefs []*logical.FieldRef
entities []tsdb.Entity
groupByEntity bool
- l *logger.Logger
}
func (i *localIndexScan) Execute(ec executor.MeasureExecutionContext) (executor.MIterator, error) {
@@ -205,11 +205,10 @@ func IndexScan(startTime, endTime time.Time, metadata *commonv1.Metadata, filter
var _ executor.MIterator = (*indexScanIterator)(nil)
type indexScanIterator struct {
- context transformContext
inner logical.ItemIterator
-
- current *measurev1.DataPoint
err error
+ current *measurev1.DataPoint
+ context transformContext
}
func newIndexScanIterator(inner logical.ItemIterator, context transformContext) executor.MIterator {
@@ -245,12 +244,11 @@ func (ism *indexScanIterator) Close() error {
var _ executor.MIterator = (*seriesIterator)(nil)
type seriesIterator struct {
- inner []tsdb.Iterator
+ err error
context transformContext
-
- index int
+ inner []tsdb.Iterator
current []*measurev1.DataPoint
- err error
+ index int
}
func newSeriesMIterator(inner []tsdb.Iterator, context transformContext) executor.MIterator {
diff --git a/pkg/query/logical/measure/measure_top.go b/pkg/query/logical/measure/measure_top.go
index 5b8766d..cc6487c 100644
--- a/pkg/query/logical/measure/measure_top.go
+++ b/pkg/query/logical/measure/measure_top.go
@@ -92,8 +92,8 @@ func (h *topHeap) Pop() interface{} {
}
type TopQueue struct {
- n int
th topHeap
+ n int
}
func NewTopQueue(n int, reverted bool) *TopQueue {
diff --git a/pkg/query/logical/measure/measure_top_test.go b/pkg/query/logical/measure/measure_top_test.go
index a004333..4bc16ec 100644
--- a/pkg/query/logical/measure/measure_top_test.go
+++ b/pkg/query/logical/measure/measure_top_test.go
@@ -35,9 +35,9 @@ func TestTopNStream(t *testing.T) {
}
tests := []struct {
name string
- fields fields
args args
wants []int64
+ fields fields
}{
{
name: "top 3",
diff --git a/pkg/query/logical/plan_orderby.go b/pkg/query/logical/plan_orderby.go
index 7db2dfc..dfe38a2 100644
--- a/pkg/query/logical/plan_orderby.go
+++ b/pkg/query/logical/plan_orderby.go
@@ -31,8 +31,8 @@ import (
)
type UnresolvedOrderBy struct {
- sort modelv1.Sort
targetIndexRuleName string
+ sort modelv1.Sort
}
func (u *UnresolvedOrderBy) Analyze(s Schema) (*OrderBy, error) {
@@ -67,13 +67,9 @@ func (u *UnresolvedOrderBy) Analyze(s Schema) (*OrderBy, error) {
}
type OrderBy struct {
- // orderByIndex describes the indexRule used to sort the elements/
- // It can be null since by default we may sort by created-time.
- Index *databasev1.IndexRule
- // while orderBySort describes the Sort direction
- Sort modelv1.Sort
- // TODO: support multiple tags. Currently only the first member will be used for sorting.
+ Index *databasev1.IndexRule
fieldRefs []*TagRef
+ Sort modelv1.Sort
}
func (o *OrderBy) Equal(other interface{}) bool {
@@ -114,7 +110,7 @@ func getRawTagValue(typedPair *modelv1.Tag) ([]byte, error) {
}
// SortedByIndex is used to test whether the given entities are sorted by the sortDirection
-// The given entities MUST satisfy both the positive check and the negative check for the reversed direction
+// The given entities MUST satisfy both the positive check and the negative check for the reversed direction.
func SortedByIndex(elements []*streamv1.Element, tagFamilyIdx, tagIdx int, sortDirection modelv1.Sort) bool {
if modelv1.Sort_SORT_UNSPECIFIED == sortDirection {
sortDirection = modelv1.Sort_SORT_ASC
diff --git a/pkg/query/logical/schema.go b/pkg/query/logical/schema.go
index 29e7c88..dac7660 100644
--- a/pkg/query/logical/schema.go
+++ b/pkg/query/logical/schema.go
@@ -39,15 +39,9 @@ type Schema interface {
}
type TagSpec struct {
- // Idx is defined as
- // 1) the field index based on the (stream/measure) schema for the underlying plans which
- // directly interact with the database and index modules,
- // 2) the projection index given by the users for those plans which can only access the data from parent plans,
- // e.g. orderBy plan uses this projection index to access the data entities (normally a projection view)
- // from the parent plan.
+ Spec *databasev1.TagSpec
TagFamilyIdx int
TagIdx int
- Spec *databasev1.TagSpec
}
func (fs *TagSpec) Equal(other *TagSpec) bool {
@@ -96,7 +90,7 @@ func (cs *CommonSchema) ShardNumber() uint32 {
return cs.Group.ResourceOpts.ShardNum
}
-// IndexDefined checks whether the field given is indexed
+// IndexDefined checks whether the field given is indexed.
func (cs *CommonSchema) IndexDefined(tagName string) (bool, *databasev1.IndexRule) {
for _, idxRule := range cs.IndexRules {
for _, tn := range idxRule.GetTags() {
diff --git a/pkg/query/logical/stream/schema.go b/pkg/query/logical/stream/schema.go
index fa5bce3..b7004d7 100644
--- a/pkg/query/logical/stream/schema.go
+++ b/pkg/query/logical/stream/schema.go
@@ -44,7 +44,7 @@ func (s *schema) EntityList() []string {
return s.common.EntityList
}
-// IndexDefined checks whether the field given is indexed
+// IndexDefined checks whether the field given is indexed.
func (s *schema) IndexDefined(tagName string) (bool, *databasev1.IndexRule) {
return s.common.IndexDefined(tagName)
}
@@ -69,7 +69,7 @@ func (s *schema) CreateTagRef(tags ...[]*logical.Tag) ([][]*logical.TagRef, erro
}
// ProjTags creates a projection view from the present streamSchema
-// with a given list of projections
+// with a given list of projections.
func (s *schema) ProjTags(refs ...[]*logical.TagRef) logical.Schema {
if len(refs) == 0 {
return nil
diff --git a/pkg/query/logical/stream/stream_analyzer.go b/pkg/query/logical/stream/stream_analyzer.go
index f4bdfd3..2beacc1 100644
--- a/pkg/query/logical/stream/stream_analyzer.go
+++ b/pkg/query/logical/stream/stream_analyzer.go
@@ -46,7 +46,7 @@ func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata)
return nil, err
}
- indexRules, err := a.metadataRepoImpl.IndexRules(context.TODO(), metadata)
+ indexRules, err := a.metadataRepoImpl.IndexRules(ctx, metadata)
if err != nil {
return nil, err
}
@@ -73,10 +73,7 @@ func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata)
func (a *Analyzer) Analyze(_ context.Context, criteria *streamv1.QueryRequest, metadata *commonv1.Metadata, s logical.Schema) (logical.Plan, error) {
// parse fields
- plan, err := parseTags(criteria, metadata, s)
- if err != nil {
- return nil, err
- }
+ plan := parseTags(criteria, metadata)
// parse orderBy
queryOrder := criteria.GetOrderBy()
@@ -106,7 +103,7 @@ func (a *Analyzer) Analyze(_ context.Context, criteria *streamv1.QueryRequest, m
//
// i.e. they are top-level sharding keys. For example, for the current skywalking's streamSchema,
// we use service_id + service_instance_id + state as the compound sharding keys.
-func parseTags(criteria *streamv1.QueryRequest, metadata *commonv1.Metadata, s logical.Schema) (logical.UnresolvedPlan, error) {
+func parseTags(criteria *streamv1.QueryRequest, metadata *commonv1.Metadata) logical.UnresolvedPlan {
timeRange := criteria.GetTimeRange()
projTags := make([][]*logical.Tag, len(criteria.GetProjection().GetTagFamilies()))
@@ -119,5 +116,5 @@ func parseTags(criteria *streamv1.QueryRequest, metadata *commonv1.Metadata, s l
}
return TagFilter(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
- criteria.Criteria, nil, projTags...), nil
+ criteria.Criteria, nil, projTags...)
}
diff --git a/pkg/query/logical/stream/stream_plan_indexscan_local.go b/pkg/query/logical/stream/stream_plan_indexscan_local.go
index 47bf70f..80bd20f 100644
--- a/pkg/query/logical/stream/stream_plan_indexscan_local.go
+++ b/pkg/query/logical/stream/stream_plan_indexscan_local.go
@@ -38,14 +38,14 @@ import (
var _ logical.Plan = (*localIndexScan)(nil)
type localIndexScan struct {
+ schema logical.Schema
+ filter index.Filter
*logical.OrderBy
- timeRange timestamp.TimeRange
- schema logical.Schema
metadata *commonv1.Metadata
+ l *logger.Logger
+ timeRange timestamp.TimeRange
projectionTagRefs [][]*logical.TagRef
entities []tsdb.Entity
- filter index.Filter
- l *logger.Logger
}
func (i *localIndexScan) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element, error) {
diff --git a/pkg/query/logical/stream/stream_plan_tag_filter.go b/pkg/query/logical/stream/stream_plan_tag_filter.go
index 798bfc7..332b999 100644
--- a/pkg/query/logical/stream/stream_plan_tag_filter.go
+++ b/pkg/query/logical/stream/stream_plan_tag_filter.go
@@ -18,6 +18,7 @@
package stream
import (
+ "errors"
"fmt"
"time"
@@ -36,12 +37,12 @@ import (
var _ logical.UnresolvedPlan = (*unresolvedTagFilter)(nil)
type unresolvedTagFilter struct {
- unresolvedOrderBy *logical.UnresolvedOrderBy
startTime time.Time
endTime time.Time
+ unresolvedOrderBy *logical.UnresolvedOrderBy
metadata *commonv1.Metadata
- projectionTags [][]*logical.Tag
criteria *modelv1.Criteria
+ projectionTags [][]*logical.Tag
}
func (uis *unresolvedTagFilter) Analyze(s logical.Schema) (logical.Plan, error) {
@@ -57,7 +58,8 @@ func (uis *unresolvedTagFilter) Analyze(s logical.Schema) (logical.Plan, error)
var err error
ctx.filter, ctx.entities, err = logical.BuildLocalFilter(uis.criteria, s, entityDict, entity)
if err != nil {
- if ge, ok := err.(*logical.GlobalIndexError); ok {
+ ge := &logical.GlobalIndexError{}
+ if errors.As(err, ge) {
ctx.globalConditions = append(ctx.globalConditions, ge.IndexRule, ge.Expr)
} else {
return nil, err
diff --git a/pkg/query/logical/tag_filter.go b/pkg/query/logical/tag_filter.go
index 5e0ef95..a5e59f2 100644
--- a/pkg/query/logical/tag_filter.go
+++ b/pkg/query/logical/tag_filter.go
@@ -24,22 +24,22 @@ import (
"github.com/pkg/errors"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
var ErrUnsupportedLogicalOperation = errors.New("unsupported logical operation")
type TagFilter interface {
fmt.Stringer
- Match(tagFamilies []*model_v1.TagFamily) (bool, error)
+ Match(tagFamilies []*modelv1.TagFamily) (bool, error)
}
-func BuildTagFilter(criteria *model_v1.Criteria, entityDict map[string]int, schema Schema, hasGlobalIndex bool) (TagFilter, error) {
+func BuildTagFilter(criteria *modelv1.Criteria, entityDict map[string]int, schema Schema, hasGlobalIndex bool) (TagFilter, error) {
if criteria == nil {
return BypassFilter, nil
}
switch criteria.GetExp().(type) {
- case *model_v1.Criteria_Condition:
+ case *modelv1.Criteria_Condition:
cond := criteria.GetCondition()
expr, err := parseExpr(cond.Value)
if err != nil {
@@ -52,7 +52,7 @@ func BuildTagFilter(criteria *model_v1.Criteria, entityDict map[string]int, sche
return BypassFilter, nil
}
return parseFilter(cond, expr)
- case *model_v1.Criteria_Le:
+ case *modelv1.Criteria_Le:
le := criteria.GetLe()
left, err := BuildTagFilter(le.Left, entityDict, schema, hasGlobalIndex)
if err != nil {
@@ -66,11 +66,11 @@ func BuildTagFilter(criteria *model_v1.Criteria, entityDict map[string]int, sche
return BypassFilter, nil
}
switch le.Op {
- case model_v1.LogicalExpression_LOGICAL_OP_AND:
+ case modelv1.LogicalExpression_LOGICAL_OP_AND:
and := newAndLogicalNode(2)
and.append(left).append(right)
return and, nil
- case model_v1.LogicalExpression_LOGICAL_OP_OR:
+ case modelv1.LogicalExpression_LOGICAL_OP_OR:
if hasGlobalIndex {
return nil, errors.WithMessage(ErrUnsupportedLogicalOperation, "global index doesn't support OR")
}
@@ -78,62 +78,66 @@ func BuildTagFilter(criteria *model_v1.Criteria, entityDict map[string]int, sche
or.append(left).append(right)
return or, nil
}
-
}
return nil, ErrInvalidCriteriaType
}
-func parseFilter(cond *model_v1.Condition, expr ComparableExpr) (TagFilter, error) {
+func parseFilter(cond *modelv1.Condition, expr ComparableExpr) (TagFilter, error) {
switch cond.Op {
- case model_v1.Condition_BINARY_OP_GT:
+ case modelv1.Condition_BINARY_OP_GT:
return newRangeTag(cond.Name, RangeOpts{
Lower: expr,
}), nil
- case model_v1.Condition_BINARY_OP_GE:
+ case modelv1.Condition_BINARY_OP_GE:
return newRangeTag(cond.Name, RangeOpts{
IncludesLower: true,
Lower: expr,
}), nil
- case model_v1.Condition_BINARY_OP_LT:
+ case modelv1.Condition_BINARY_OP_LT:
return newRangeTag(cond.Name, RangeOpts{
Upper: expr,
}), nil
- case model_v1.Condition_BINARY_OP_LE:
+ case modelv1.Condition_BINARY_OP_LE:
return newRangeTag(cond.Name, RangeOpts{
IncludesUpper: true,
Upper: expr,
}), nil
- case model_v1.Condition_BINARY_OP_EQ:
+ case modelv1.Condition_BINARY_OP_EQ:
return newEqTag(cond.Name, expr), nil
- case model_v1.Condition_BINARY_OP_NE:
+ case modelv1.Condition_BINARY_OP_NE:
return newNotTag(newEqTag(cond.Name, expr)), nil
- case model_v1.Condition_BINARY_OP_HAVING:
+ case modelv1.Condition_BINARY_OP_HAVING:
return newHavingTag(cond.Name, expr), nil
- case model_v1.Condition_BINARY_OP_NOT_HAVING:
+ case modelv1.Condition_BINARY_OP_NOT_HAVING:
return newNotTag(newHavingTag(cond.Name, expr)), nil
+ case modelv1.Condition_BINARY_OP_IN:
+ panic("unimplemented")
+ case modelv1.Condition_BINARY_OP_NOT_IN:
+ panic("unimplemented")
+ default:
+ return nil, errors.WithMessagef(ErrUnsupportedConditionOp, "tag filter parses %v", cond)
}
- return nil, errors.WithMessagef(ErrUnsupportedConditionOp, "tag filter parses %v", cond)
}
-func parseExpr(value *model_v1.TagValue) (ComparableExpr, error) {
+func parseExpr(value *modelv1.TagValue) (ComparableExpr, error) {
switch v := value.Value.(type) {
- case *model_v1.TagValue_Str:
+ case *modelv1.TagValue_Str:
return &strLiteral{v.Str.GetValue()}, nil
- case *model_v1.TagValue_Id:
+ case *modelv1.TagValue_Id:
return &idLiteral{v.Id.GetValue()}, nil
- case *model_v1.TagValue_StrArray:
+ case *modelv1.TagValue_StrArray:
return &strArrLiteral{
arr: v.StrArray.GetValue(),
}, nil
- case *model_v1.TagValue_Int:
+ case *modelv1.TagValue_Int:
return &int64Literal{
int64: v.Int.GetValue(),
}, nil
- case *model_v1.TagValue_IntArray:
+ case *modelv1.TagValue_IntArray:
return &int64ArrLiteral{
arr: v.IntArray.GetValue(),
}, nil
- case *model_v1.TagValue_Null:
+ case *modelv1.TagValue_Null:
return nullLiteralExpr, nil
}
return nil, errors.WithMessagef(ErrUnsupportedConditionValue, "tag filter parses %v", value)
@@ -143,7 +147,7 @@ var BypassFilter = new(emptyFilter)
type emptyFilter struct{}
-func (emptyFilter) Match(_ []*model_v1.TagFamily) (bool, error) { return true, nil }
+func (emptyFilter) Match(_ []*modelv1.TagFamily) (bool, error) { return true, nil }
func (emptyFilter) String() string { return "bypass" }
@@ -164,7 +168,7 @@ func (n *logicalNode) append(sub TagFilter) *logicalNode {
return n
}
-func matchTag(tagFamilies []*model_v1.TagFamily, n *logicalNode, lp logicalNodeOP) (bool, error) {
+func matchTag(tagFamilies []*modelv1.TagFamily, n *logicalNode, lp logicalNodeOP) (bool, error) {
var result *bool
for _, sn := range n.SubNodes {
r, err := sn.Match(tagFamilies)
@@ -202,7 +206,7 @@ func (an *andLogicalNode) merge(bb ...bool) bool {
return true
}
-func (an *andLogicalNode) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (an *andLogicalNode) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
return matchTag(tagFamilies, an.logicalNode, an)
}
@@ -237,7 +241,7 @@ func (on *orLogicalNode) merge(bb ...bool) bool {
return false
}
-func (on *orLogicalNode) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (on *orLogicalNode) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
return matchTag(tagFamilies, on.logicalNode, on)
}
@@ -253,8 +257,8 @@ func (on *orLogicalNode) String() string {
type tagLeaf struct {
TagFilter
- Name string
Expr LiteralExpr
+ Name string
}
func (l *tagLeaf) MarshalJSON() ([]byte, error) {
@@ -275,7 +279,7 @@ func newNotTag(inner TagFilter) *notTag {
}
}
-func (n *notTag) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (n *notTag) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
b, err := n.Inner.Match(tagFamilies)
if err != nil {
return false, err
@@ -306,7 +310,7 @@ func newEqTag(tagName string, values LiteralExpr) *eqTag {
}
}
-func (eq *eqTag) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (eq *eqTag) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
expr, err := tagExpr(tagFamilies, eq.Name)
if err != nil {
return false, err
@@ -345,7 +349,7 @@ func newRangeTag(tagName string, opts RangeOpts) *rangeTag {
}
}
-func (r *rangeTag) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (r *rangeTag) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
expr, err := tagExpr(tagFamilies, r.Name)
if err != nil {
return false, err
@@ -414,7 +418,7 @@ func (r *rangeTag) String() string {
return jsonToString(r)
}
-func tagExpr(tagFamilies []*model_v1.TagFamily, tagName string) (ComparableExpr, error) {
+func tagExpr(tagFamilies []*modelv1.TagFamily, tagName string) (ComparableExpr, error) {
for _, tf := range tagFamilies {
for _, t := range tf.Tags {
if t.Key == tagName {
@@ -438,7 +442,7 @@ func newHavingTag(tagName string, values LiteralExpr) *havingTag {
}
}
-func (h *havingTag) Match(tagFamilies []*model_v1.TagFamily) (bool, error) {
+func (h *havingTag) Match(tagFamilies []*modelv1.TagFamily) (bool, error) {
expr, err := tagExpr(tagFamilies, h.Name)
if err != nil {
return false, err
diff --git a/pkg/run/closer.go b/pkg/run/closer.go
index 34a87bb..41d4205 100644
--- a/pkg/run/closer.go
+++ b/pkg/run/closer.go
@@ -24,12 +24,11 @@ import (
// Closer can close a goroutine then wait for it to stop.
type Closer struct {
+ ctx context.Context
+ cancel context.CancelFunc
waiting sync.WaitGroup
lock sync.RWMutex
closed bool
-
- ctx context.Context
- cancel context.CancelFunc
}
// NewCloser instances a new Closer.
@@ -70,7 +69,7 @@ func (c *Closer) CloseThenWait() {
c.waiting.Wait()
}
-// Closed returns whether the Closer is closed
+// Closed returns whether the Closer is closed.
func (c *Closer) Closed() bool {
c.lock.RLock()
defer c.lock.RUnlock()
diff --git a/pkg/run/run.go b/pkg/run/run.go
index 1826d2b..5547b6b 100644
--- a/pkg/run/run.go
+++ b/pkg/run/run.go
@@ -82,8 +82,8 @@ func NewPreRunner(name string, fn func() error) PreRunner {
}
type preRunner struct {
- name string
fn func() error
+ name string
}
func (p preRunner) Name() string {
@@ -118,19 +118,16 @@ type Service interface {
// to manage service lifecycles. It allows for easy composition of elegant
// monoliths as well as adding signal handlers, metrics services, etc.
type Group struct {
- name string
-
- f *FlagSet
- r run.Group
- c []Config
- p []PreRunner
- s []Service
- readyCh chan struct{}
- log *logger.Logger
-
+ f *FlagSet
+ readyCh chan struct{}
+ log *logger.Logger
+ name string
+ r run.Group
+ c []Config
+ p []PreRunner
+ s []Service
showRunGroup bool
-
- configured bool
+ configured bool
}
func NewGroup(name string) Group {
@@ -140,7 +137,7 @@ func NewGroup(name string) Group {
}
}
-// Name shows the name of the group
+// Name shows the name of the group.
func (g Group) Name() string {
return g.name
}
@@ -150,7 +147,7 @@ func (g Group) Name() string {
// phases. If a Unit doesn't satisfy any of the bootstrap phases it is ignored
// by Group.
// The returned array of booleans is of the same size as the amount of provided
-// Units, signalling for each provided Unit if it successfully registered with
+// Units, signaling for each provided Unit if it successfully registered with
// Group for at least one of the bootstrap phases or if it was ignored.
func (g *Group) Register(units ...Unit) []bool {
g.log = logger.GetLogger(g.name)
diff --git a/pkg/run/test.go b/pkg/run/test.go
index 095744c..7151c72 100644
--- a/pkg/run/test.go
+++ b/pkg/run/test.go
@@ -25,15 +25,15 @@ import (
var _ Service = (*Tester)(nil)
type Tester struct {
- ID string
startedNotifier chan struct{}
stopCh chan struct{}
+ ID string
once sync.Once
}
-func NewTester(ID string) *Tester {
+func NewTester(id string) *Tester {
return &Tester{
- ID: ID,
+ ID: id,
startedNotifier: make(chan struct{}),
stopCh: make(chan struct{}),
once: sync.Once{},
diff --git a/pkg/schema/metadata.go b/pkg/schema/metadata.go
index f48b237..1b2f979 100644
--- a/pkg/schema/metadata.go
+++ b/pkg/schema/metadata.go
@@ -61,9 +61,9 @@ type Group interface {
}
type MetadataEvent struct {
+ Metadata *commonv1.Metadata
Typ EventType
Kind EventKind
- Metadata *commonv1.Metadata
}
type ResourceSchema interface {
@@ -105,18 +105,16 @@ type Repository interface {
var _ Repository = (*schemaRepo)(nil)
type schemaRepo struct {
- sync.RWMutex
metadata metadata.Repo
repo discovery.ServiceRepo
- l *logger.Logger
resourceSupplier ResourceSupplier
+ l *logger.Logger
+ data map[string]*group
+ workerStopCh chan struct{}
+ eventCh chan MetadataEvent
shardTopic bus.Topic
entityTopic bus.Topic
- data map[string]*group
-
- // stop channel for the inner worker
- workerStopCh chan struct{}
- eventCh chan MetadataEvent
+ sync.RWMutex
}
func NewRepository(
@@ -375,15 +373,15 @@ func (sr *schemaRepo) Close() {
var _ Group = (*group)(nil)
type group struct {
- groupSchema atomic.Pointer[commonv1.Group]
- l *logger.Logger
resourceSupplier ResourceSupplier
repo discovery.ServiceRepo
metadata metadata.Repo
- entityTopic bus.Topic
db atomic.Value
- mapMutex sync.RWMutex
+ groupSchema atomic.Pointer[commonv1.Group]
+ l *logger.Logger
schemaMap map[string]Resource
+ entityTopic bus.Topic
+ mapMutex sync.RWMutex
}
func newGroup(
diff --git a/pkg/test/flow/slice.go b/pkg/test/flow/slice.go
index 1792d2e..9d6a4fd 100644
--- a/pkg/test/flow/slice.go
+++ b/pkg/test/flow/slice.go
@@ -29,9 +29,9 @@ import (
var _ flow.Source = (*sourceSlice)(nil)
type sourceSlice struct {
- flow.ComponentState
slice interface{}
out chan flow.StreamRecord
+ flow.ComponentState
}
func (s *sourceSlice) Out() <-chan flow.StreamRecord {
diff --git a/pkg/test/helpers/context.go b/pkg/test/helpers/context.go
index f2250c8..b283903 100644
--- a/pkg/test/helpers/context.go
+++ b/pkg/test/helpers/context.go
@@ -27,7 +27,7 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"sigs.k8s.io/yaml"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
type SharedContext struct {
@@ -36,12 +36,12 @@ type SharedContext struct {
}
type Args struct {
- Input string
- Offset time.Duration
- Duration time.Duration
Begin *timestamppb.Timestamp
End *timestamppb.Timestamp
+ Input string
Want string
+ Offset time.Duration
+ Duration time.Duration
WantEmpty bool
WantErr bool
}
@@ -52,15 +52,15 @@ func UnmarshalYAML(ii []byte, m proto.Message) {
gomega.Expect(protojson.Unmarshal(j, m)).To(gomega.Succeed())
}
-func TimeRange(args Args, shardContext SharedContext) *model_v1.TimeRange {
+func TimeRange(args Args, shardContext SharedContext) *modelv1.TimeRange {
if args.Begin != nil && args.End != nil {
- return &model_v1.TimeRange{
+ return &modelv1.TimeRange{
Begin: args.Begin,
End: args.End,
}
}
b := shardContext.BaseTime.Add(args.Offset)
- return &model_v1.TimeRange{
+ return &modelv1.TimeRange{
Begin: timestamppb.New(b),
End: timestamppb.New(b.Add(args.Duration)),
}
diff --git a/pkg/test/measure/traffic/traffic.go b/pkg/test/measure/traffic/traffic.go
index e932f4a..0b40479 100644
--- a/pkg/test/measure/traffic/traffic.go
+++ b/pkg/test/measure/traffic/traffic.go
@@ -20,6 +20,7 @@ package traffic
import (
"context"
"crypto/rand"
+ "errors"
"fmt"
"io"
"strconv"
@@ -88,12 +89,12 @@ func SendWrites(ts TestCase) (*z.Closer, error) {
}
type sender struct {
+ client measurev1.MeasureService_WriteClient
svc string
instance string
succeed uint64
sendFailed uint64
revFailed uint64
- client measurev1.MeasureService_WriteClient
metricNum int
}
@@ -171,7 +172,7 @@ func (s *sender) write() {
return
}
_, err = s.client.Recv()
- if err != nil && err != io.EOF {
+ if errors.Is(err, io.EOF) {
l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("receiving failed")
s.revFailed++
return
diff --git a/pkg/test/setup/setup.go b/pkg/test/setup/setup.go
index ee5b011..4235604 100644
--- a/pkg/test/setup/setup.go
+++ b/pkg/test/setup/setup.go
@@ -103,8 +103,8 @@ func modules(flags []string) func() {
}
type preloadService struct {
- name string
metaSvc metadata.Service
+ name string
}
func (p *preloadService) Name() string {
diff --git a/pkg/test/stream/traffic/traffic.go b/pkg/test/stream/traffic/traffic.go
index eced584..7ffe143 100644
--- a/pkg/test/stream/traffic/traffic.go
+++ b/pkg/test/stream/traffic/traffic.go
@@ -20,9 +20,10 @@ package traffic
import (
"context"
"crypto/rand"
- // Load some tag templates
+ // Load some tag templates.
_ "embed"
"encoding/base64"
+ "errors"
"fmt"
"io"
"strconv"
@@ -107,13 +108,13 @@ func SendWrites(ts TestCase) (*z.Closer, error) {
}
type sender struct {
+ client streamv1.StreamService_WriteClient
svc string
instance string
+ data []byte
succeed uint64
sendFailed uint64
revFailed uint64
- client streamv1.StreamService_WriteClient
- data []byte
}
func newSender(svcID, instanceID int, client streamv1.StreamServiceClient, data []byte) (*sender, error) {
@@ -180,7 +181,7 @@ func (s *sender) write(searchTagFamily *modelv1.TagFamilyForWrite) {
return
}
_, err = s.client.Recv()
- if err != nil && err != io.EOF {
+ if errors.Is(err, io.EOF) {
l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("receiving failed")
s.revFailed++
return
diff --git a/pkg/timestamp/clock.go b/pkg/timestamp/clock.go
index 322747a..1746e2a 100644
--- a/pkg/timestamp/clock.go
+++ b/pkg/timestamp/clock.go
@@ -64,7 +64,7 @@ func GetClock(ctx context.Context) (Clock, context.Context) {
return c.(Clock), ctx
}
-// SetContext returns a sub context with the passed Clock
+// SetContext returns a sub context with the passed Clock.
func SetClock(ctx context.Context, clock Clock) context.Context {
return context.WithValue(ctx, clockKey, clock)
}
diff --git a/pkg/timestamp/nano.go b/pkg/timestamp/nano.go
index 3628573..bdd3bbb 100644
--- a/pkg/timestamp/nano.go
+++ b/pkg/timestamp/nano.go
@@ -20,7 +20,7 @@ package timestamp
import (
"math"
"time"
- // link runtime pkg fastrand
+ // link runtime pkg fastrand.
_ "unsafe"
"github.com/pkg/errors"
@@ -39,7 +39,7 @@ var (
//go:linkname fastRandN runtime.fastrandn
func fastRandN(n uint32) uint32
-// MToN convert time unix millisends to nanoseconds
+// MToN convert time unix millisends to nanoseconds.
func MToN(ms time.Time) time.Time {
ns := ms.UnixNano()
if ms.Nanosecond()%int(mSecond) > 0 {
@@ -49,7 +49,7 @@ func MToN(ms time.Time) time.Time {
return time.Unix(ms.Unix(), nns%int64(time.Second))
}
-// NowMilli returns a time based on a unix millisecond
+// NowMilli returns a time based on a unix millisecond.
func NowMilli() time.Time {
return time.UnixMilli(time.Now().UnixMilli())
}
@@ -57,12 +57,12 @@ func NowMilli() time.Time {
const (
// MinNanoTime is the minimum time that can be represented.
//
- // 1677-09-21 00:12:43.145224192 +0000 UTC
+ // 1677-09-21 00:12:43.145224192 +0000 UTC.
MinNanoTime = int64(math.MinInt64)
// MaxNanoTime is the maximum time that can be represented.
//
- // 2262-04-11 23:47:16.854775807 +0000 UTC
+ // 2262-04-11 23:47:16.854775807 +0000 UTC.
MaxNanoTime = int64(math.MaxInt64)
)
@@ -83,7 +83,7 @@ var (
ErrTimeEmpty = errors.Errorf("time is empty")
)
-// Check checks that a time is valid
+// Check checks that a time is valid.
func Check(t time.Time) error {
if t.Before(minNanoTime) || t.After(maxNanoTime) {
return ErrTimeOutOfRange
@@ -94,7 +94,7 @@ func Check(t time.Time) error {
return nil
}
-// CheckPb checks that a protobuf timestamp is valid
+// CheckPb checks that a protobuf timestamp is valid.
func CheckPb(t *timestamppb.Timestamp) error {
if t == nil {
return ErrTimeEmpty
@@ -102,7 +102,7 @@ func CheckPb(t *timestamppb.Timestamp) error {
return Check(t.AsTime())
}
-// CheckTimeRange checks that a protobuf time range is valid
+// CheckTimeRange checks that a protobuf time range is valid.
func CheckTimeRange(timeRange *modelv1.TimeRange) error {
if timeRange == nil {
return ErrTimeEmpty
diff --git a/pkg/timestamp/scheduler.go b/pkg/timestamp/scheduler.go
index 60a6eac..15e0e38 100644
--- a/pkg/timestamp/scheduler.go
+++ b/pkg/timestamp/scheduler.go
@@ -35,22 +35,21 @@ var (
)
// SchedulerAction is an executable when a trigger is fired
-// now is the trigger time, logger has a context indicating the task's identity
+// now is the trigger time, logger has a context indicating the task's identity.
type SchedulerAction func(now time.Time, logger *logger.Logger) bool
// Scheduler maintains a registry of tasks and their duty cycle.
// It also provides a Trigger method to fire a task that is scheduled by a MockClock.
type Scheduler struct {
+ clock Clock
+ l *logger.Logger
+ tasks map[string]*task
sync.RWMutex
- clock Clock
isMock bool
- l *logger.Logger
-
- tasks map[string]*task
closed bool
}
-// NewScheduler returns an instance of Scheduler
+// NewScheduler returns an instance of Scheduler.
func NewScheduler(parent *logger.Logger, clock Clock) *Scheduler {
var isMock bool
if _, ok := clock.(MockClock); ok {
@@ -65,7 +64,7 @@ func NewScheduler(parent *logger.Logger, clock Clock) *Scheduler {
}
// Register adds the given task's SchedulerAction to the Scheduler,
-// and associate the given schedule expression
+// and associate the given schedule expression.
func (s *Scheduler) Register(name string, options cron.ParseOption, expr string, action SchedulerAction) error {
s.Lock()
defer s.Unlock()
@@ -120,14 +119,14 @@ func (s *Scheduler) Trigger(name string) bool {
return true
}
-// Closed returns whether the Scheduler is closed
+// Closed returns whether the Scheduler is closed.
func (s *Scheduler) Closed() bool {
s.RLock()
defer s.RUnlock()
return s.closed
}
-// Close the Scheduler and shut down all registered tasks
+// Close the Scheduler and shut down all registered tasks.
func (s *Scheduler) Close() {
s.Lock()
defer s.Unlock()
diff --git a/pkg/version/version.go b/pkg/version/version.go
index 10dc5cb..21827cb 100644
--- a/pkg/version/version.go
+++ b/pkg/version/version.go
@@ -27,17 +27,17 @@ import (
// build is to be populated at build time using -ldflags -X.
var build string
-// Show the service's build information
+// Show the service's build information.
func Build() string {
return build
}
-// Show the service's version information
+// Show the service's version information.
func Show(serviceName string) {
fmt.Println(serviceName + " " + Parse())
}
-// Parse returns the parsed service's version information. (from raw git label)
+// Parse returns the parsed service's version information. (from raw git label).
func Parse() string {
// versionString syntax:
// <release tag>-<commits since release tag>-g<commit hash>-<branch name>
diff --git a/revive.toml b/revive.toml
index 0cae753..6e083f1 100644
--- a/revive.toml
+++ b/revive.toml
@@ -38,3 +38,8 @@ warningCode = 1
[rule.unexported-return]
[rule.indent-error-flow]
[rule.errorf]
+[rule.empty-block]
+[rule.superfluous-else]
+[rule.unused-parameter]
+[rule.unreachable-code]
+[rule.redefines-builtin-id]
diff --git a/scripts/build/lint-api.mk b/scripts/build/lint-api.mk
index 3d3a243..5bc3bba 100644
--- a/scripts/build/lint-api.mk
+++ b/scripts/build/lint-api.mk
@@ -16,16 +16,15 @@
# under the License.
#
-LINT_OPTS ?= --timeout 1m0s
-
include $(mk_dir)lint-bin.mk
##@ Code quality targets
.PHONY: lint
-lint: $(LINTER)
+lint: $(LINTER) $(REVIVE) ## Run all linters
@PATH=$(tool_bin):$(proto_dir) $(BUF) lint
- $(LINTER) --verbose run $(LINT_OPTS) --config $(root_dir)/deprecated-golangci.yml
+ $(LINTER) run -v --config $(root_dir)/.golangci.yml ./... && \
+ $(REVIVE) -config $(root_dir)/revive.toml -formatter friendly ./...
.PHONY: format
format: $(BUF)
diff --git a/scripts/build/lint-bin.mk b/scripts/build/lint-bin.mk
index 427ffce..ef50dcd 100644
--- a/scripts/build/lint-bin.mk
+++ b/scripts/build/lint-bin.mk
@@ -1,4 +1,25 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
LINTER := $(tool_bin)/golangci-lint
$(LINTER):
@GOBIN=$(tool_bin) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+
+REVIVE := $(tool_bin)/revive
+$(REVIVE):
+ @GOBIN=$(tool_bin) go install github.com/mgechev/revive@$(REVIVE_VERSION)
diff --git a/scripts/build/lint-deprecated.mk b/scripts/build/lint-deprecated.mk
deleted file mode 100644
index e9ee439..0000000
--- a/scripts/build/lint-deprecated.mk
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to Apache Software Foundation (ASF) under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Apache Software Foundation (ASF) licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-LINT_OPTS ?= --timeout 1m0s
-
-include $(mk_dir)lint-bin.mk
-
-##@ Code quality targets
-
-.PHONY: lint
-lint: $(LINTER)
- $(LINTER) --verbose run $(LINT_OPTS) --config $(root_dir)/deprecated-golangci.yml
-
-.PHONY: format
-format: $(LINTER)
- $(LINTER) run --fix -c $(root_dir)/.golangci-format.yml
diff --git a/scripts/build/lint.mk b/scripts/build/lint.mk
index f7c81b8..453d2d3 100644
--- a/scripts/build/lint.mk
+++ b/scripts/build/lint.mk
@@ -20,9 +20,7 @@ include $(mk_dir)lint-bin.mk
##@ Code quality targets
-REVIVE := $(tool_bin)/revive
-$(REVIVE):
- @GOBIN=$(tool_bin) go install github.com/mgechev/revive@$(REVIVE_VERSION)
+
.PHONY: lint
lint: $(LINTER) $(REVIVE) ## Run all linters
diff --git a/test/cases/measure/data/data.go b/test/cases/measure/data/data.go
index 7cf31ec..04316ef 100644
--- a/test/cases/measure/data/data.go
+++ b/test/cases/measure/data/data.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package data contains integration test cases of the measure
+// Package data contains integration test cases of the measure.
package data
import (
@@ -46,7 +46,7 @@ var inputFS embed.FS
//go:embed want/*.yaml
var wantFS embed.FS
-// VerifyFn verify whether the query response matches the wanted result
+// VerifyFn verify whether the query response matches the wanted result.
var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args helpers.Args) {
i, err := inputFS.ReadFile("input/" + args.Input + ".yaml")
innerGm.Expect(err).NotTo(gm.HaveOccurred())
@@ -110,7 +110,7 @@ func loadData(md *commonv1.Metadata, measure measurev1.MeasureService_WriteClien
}
}
-// Write data into the server
+// Write data into the server.
func Write(conn *grpclib.ClientConn, name, group, dataFile string,
baseTime time.Time, interval time.Duration,
) {
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index 8af9ca5..02bcc42 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package measure_test contains integration test cases of the measure
+// Package measure_test contains integration test cases of the measure.
package measure_test
import (
@@ -30,7 +30,7 @@ import (
)
var (
- // SharedContext is the parallel execution context
+ // SharedContext is the parallel execution context.
SharedContext helpers.SharedContext
verify = func(args helpers.Args) {
gm.Eventually(func(innerGm gm.Gomega) {
diff --git a/test/cases/stream/data/data.go b/test/cases/stream/data/data.go
index 153141b..cb6cbde 100644
--- a/test/cases/stream/data/data.go
+++ b/test/cases/stream/data/data.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package data contains integration test cases of the stream
+// Package data contains integration test cases of the stream.
package data
import (
@@ -36,9 +36,9 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"sigs.k8s.io/yaml"
- common_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- stream_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
)
@@ -52,14 +52,14 @@ var wantFS embed.FS
//go:embed testdata/*.json
var dataFS embed.FS
-// VerifyFn verify whether the query response matches the wanted result
+// VerifyFn verify whether the query response matches the wanted result.
var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args helpers.Args) {
i, err := inputFS.ReadFile("input/" + args.Input + ".yaml")
innerGm.Expect(err).NotTo(gm.HaveOccurred())
- query := &stream_v1.QueryRequest{}
+ query := &streamv1.QueryRequest{}
helpers.UnmarshalYAML(i, query)
query.TimeRange = helpers.TimeRange(args, sharedContext)
- c := stream_v1.NewStreamServiceClient(sharedContext.Connection)
+ c := streamv1.NewStreamServiceClient(sharedContext.Connection)
ctx := context.Background()
resp, err := c.Query(ctx, query)
if args.WantErr {
@@ -78,11 +78,11 @@ var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args
}
ww, err := wantFS.ReadFile("want/" + args.Want + ".yaml")
innerGm.Expect(err).NotTo(gm.HaveOccurred())
- want := &stream_v1.QueryResponse{}
+ want := &streamv1.QueryResponse{}
helpers.UnmarshalYAML(ww, want)
innerGm.Expect(cmp.Equal(resp, want,
protocmp.IgnoreUnknown(),
- protocmp.IgnoreFields(&stream_v1.Element{}, "timestamp"),
+ protocmp.IgnoreFields(&streamv1.Element{}, "timestamp"),
protocmp.Transform())).
To(gm.BeTrue(), func() string {
j, err := protojson.Marshal(resp)
@@ -97,7 +97,7 @@ var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args
})
}
-func loadData(stream stream_v1.StreamService_WriteClient, dataFile string, baseTime time.Time, interval time.Duration) {
+func loadData(stream streamv1.StreamService_WriteClient, dataFile string, baseTime time.Time, interval time.Duration) {
var templates []interface{}
content, err := dataFS.ReadFile("testdata/" + dataFile)
gm.Expect(err).ShouldNot(gm.HaveOccurred())
@@ -106,16 +106,16 @@ func loadData(stream stream_v1.StreamService_WriteClient, dataFile string, baseT
for i, template := range templates {
rawSearchTagFamily, errMarshal := json.Marshal(template)
gm.Expect(errMarshal).ShouldNot(gm.HaveOccurred())
- searchTagFamily := &model_v1.TagFamilyForWrite{}
+ searchTagFamily := &modelv1.TagFamilyForWrite{}
gm.Expect(protojson.Unmarshal(rawSearchTagFamily, searchTagFamily)).ShouldNot(gm.HaveOccurred())
- e := &stream_v1.ElementValue{
+ e := &streamv1.ElementValue{
ElementId: strconv.Itoa(i),
Timestamp: timestamppb.New(baseTime.Add(interval * time.Duration(i))),
- TagFamilies: []*model_v1.TagFamilyForWrite{
+ TagFamilies: []*modelv1.TagFamilyForWrite{
{
- Tags: []*model_v1.TagValue{
+ Tags: []*modelv1.TagValue{
{
- Value: &model_v1.TagValue_BinaryData{
+ Value: &modelv1.TagValue_BinaryData{
BinaryData: bb,
},
},
@@ -124,8 +124,8 @@ func loadData(stream stream_v1.StreamService_WriteClient, dataFile string, baseT
},
}
e.TagFamilies = append(e.TagFamilies, searchTagFamily)
- errInner := stream.Send(&stream_v1.WriteRequest{
- Metadata: &common_v1.Metadata{
+ errInner := stream.Send(&streamv1.WriteRequest{
+ Metadata: &commonv1.Metadata{
Name: "sw",
Group: "default",
},
@@ -135,9 +135,9 @@ func loadData(stream stream_v1.StreamService_WriteClient, dataFile string, baseT
}
}
-// Write data into the server
+// Write data into the server.
func Write(conn *grpclib.ClientConn, dataFile string, baseTime time.Time, interval time.Duration) {
- c := stream_v1.NewStreamServiceClient(conn)
+ c := streamv1.NewStreamServiceClient(conn)
ctx := context.Background()
writeClient, err := c.Write(ctx)
gm.Expect(err).NotTo(gm.HaveOccurred())
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index 9f64e3a..4ac8cde 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package stream_test contains integration test cases of the stream
+// Package stream_test contains integration test cases of the stream.
package stream_test
import (
@@ -32,7 +32,7 @@ import (
)
var (
- // SharedContext is the parallel execution context
+ // SharedContext is the parallel execution context.
SharedContext helpers.SharedContext
verify = func(innerGm gm.Gomega, args helpers.Args) {
gm.Eventually(func(innerGm gm.Gomega) {
diff --git a/test/cases/topn/data/data.go b/test/cases/topn/data/data.go
index 48fb0c2..1eb7fbb 100644
--- a/test/cases/topn/data/data.go
+++ b/test/cases/topn/data/data.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package data contains integration test cases of the topN
+// Package data contains integration test cases of the topN.
package data
import (
@@ -39,7 +39,7 @@ var inputFS embed.FS
//go:embed want/*.yaml
var wantFS embed.FS
-// VerifyFn verify whether the query response matches the wanted result
+// VerifyFn verify whether the query response matches the wanted result.
var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args helpers.Args) {
i, err := inputFS.ReadFile("input/" + args.Input + ".yaml")
innerGm.Expect(err).NotTo(gm.HaveOccurred())
diff --git a/test/cases/topn/topn.go b/test/cases/topn/topn.go
index e23e4c7..3147048 100644
--- a/test/cases/topn/topn.go
+++ b/test/cases/topn/topn.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package topn_test contains integration test cases of the TopN
+// Package topn_test contains integration test cases of the TopN.
package topn_test
import (
@@ -30,7 +30,7 @@ import (
)
var (
- // SharedContext is the parallel execution context
+ // SharedContext is the parallel execution context.
SharedContext helpers.SharedContext
verify = func(args helpers.Args) {
gm.Eventually(func(innerGm gm.Gomega) {
diff --git a/test/e2e-v2/cases/profiling/ebpf/oncpu/sqrt.go b/test/e2e-v2/cases/profiling/ebpf/oncpu/sqrt.go
index 7e3189a..eb3dbc0 100644
--- a/test/e2e-v2/cases/profiling/ebpf/oncpu/sqrt.go
+++ b/test/e2e-v2/cases/profiling/ebpf/oncpu/sqrt.go
@@ -14,7 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Package main is the entry point of the process for EBPF profiling
+// Package main is the entry point of the process for EBPF profiling.
package main
import (
diff --git a/test/integration/other/property_test.go b/test/integration/other/property_test.go
index e08e260..fa1f9d2 100644
--- a/test/integration/other/property_test.go
+++ b/test/integration/other/property_test.go
@@ -26,9 +26,9 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
- common_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- model_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- property_v1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
@@ -38,7 +38,7 @@ import (
var _ = Describe("Property application", func() {
var deferFn func()
var conn *grpc.ClientConn
- var client property_v1.PropertyServiceClient
+ var client propertyv1.PropertyServiceClient
BeforeEach(func() {
var addr string
@@ -48,50 +48,50 @@ var _ = Describe("Property application", func() {
var err error
conn, err = grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
Expect(err).NotTo(HaveOccurred())
- client = property_v1.NewPropertyServiceClient(conn)
+ client = propertyv1.NewPropertyServiceClient(conn)
})
AfterEach(func() {
Expect(conn.Close()).To(Succeed())
deferFn()
})
It("applies properties", func() {
- md := &property_v1.Metadata{
- Container: &common_v1.Metadata{
+ md := &propertyv1.Metadata{
+ Container: &commonv1.Metadata{
Name: "p",
Group: "g",
},
Id: "1",
}
- resp, err := client.Apply(context.Background(), &property_v1.ApplyRequest{Property: &property_v1.Property{
+ resp, err := client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
Metadata: md,
- Tags: []*model_v1.Tag{
- {Key: "t1", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v2"}}}},
+ Tags: []*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
},
}})
Expect(err).NotTo(HaveOccurred())
Expect(resp.Created).To(BeTrue())
Expect(resp.TagsNum).To(Equal(uint32(2)))
- got, err := client.Get(context.Background(), &property_v1.GetRequest{Metadata: md})
+ got, err := client.Get(context.Background(), &propertyv1.GetRequest{Metadata: md})
Expect(err).NotTo(HaveOccurred())
- Expect(got.Property.Tags).To(Equal([]*model_v1.Tag{
- {Key: "t1", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v2"}}}},
+ Expect(got.Property.Tags).To(Equal([]*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
}))
- resp, err = client.Apply(context.Background(), &property_v1.ApplyRequest{Property: &property_v1.Property{
+ resp, err = client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
Metadata: md,
- Tags: []*model_v1.Tag{
- {Key: "t2", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v22"}}}},
+ Tags: []*modelv1.Tag{
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v22"}}}},
},
}})
Expect(err).NotTo(HaveOccurred())
Expect(resp.Created).To(BeFalse())
Expect(resp.TagsNum).To(Equal(uint32(1)))
- got, err = client.Get(context.Background(), &property_v1.GetRequest{Metadata: md})
+ got, err = client.Get(context.Background(), &propertyv1.GetRequest{Metadata: md})
Expect(err).NotTo(HaveOccurred())
- Expect(got.Property.Tags).To(Equal([]*model_v1.Tag{
- {Key: "t1", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v22"}}}},
+ Expect(got.Property.Tags).To(Equal([]*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v22"}}}},
}))
})
})
@@ -99,8 +99,8 @@ var _ = Describe("Property application", func() {
var _ = Describe("Property application", func() {
var deferFn func()
var conn *grpc.ClientConn
- var client property_v1.PropertyServiceClient
- var md *property_v1.Metadata
+ var client propertyv1.PropertyServiceClient
+ var md *propertyv1.Metadata
BeforeEach(func() {
var addr string
@@ -108,19 +108,19 @@ var _ = Describe("Property application", func() {
var err error
conn, err = grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
Expect(err).NotTo(HaveOccurred())
- client = property_v1.NewPropertyServiceClient(conn)
- md = &property_v1.Metadata{
- Container: &common_v1.Metadata{
+ client = propertyv1.NewPropertyServiceClient(conn)
+ md = &propertyv1.Metadata{
+ Container: &commonv1.Metadata{
Name: "p",
Group: "g",
},
Id: "1",
}
- resp, err := client.Apply(context.Background(), &property_v1.ApplyRequest{Property: &property_v1.Property{
+ resp, err := client.Apply(context.Background(), &propertyv1.ApplyRequest{Property: &propertyv1.Property{
Metadata: md,
- Tags: []*model_v1.Tag{
- {Key: "t1", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v1"}}}},
- {Key: "t2", Value: &model_v1.TagValue{Value: &model_v1.TagValue_Str{Str: &model_v1.Str{Value: "v2"}}}},
+ Tags: []*modelv1.Tag{
+ {Key: "t1", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v1"}}}},
+ {Key: "t2", Value: &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: "v2"}}}},
},
}})
Expect(err).NotTo(HaveOccurred())
@@ -132,15 +132,15 @@ var _ = Describe("Property application", func() {
deferFn()
})
It("lists properties", func() {
- got, err := client.List(context.Background(), &property_v1.ListRequest{Container: md.Container})
+ got, err := client.List(context.Background(), &propertyv1.ListRequest{Container: md.Container})
Expect(err).NotTo(HaveOccurred())
Expect(len(got.Property)).To(Equal(1))
})
It("deletes properties", func() {
- got, err := client.Delete(context.Background(), &property_v1.DeleteRequest{Metadata: md})
+ got, err := client.Delete(context.Background(), &propertyv1.DeleteRequest{Metadata: md})
Expect(err).NotTo(HaveOccurred())
Expect(got.Deleted).To(BeTrue())
- _, err = client.Get(context.Background(), &property_v1.GetRequest{Metadata: md})
+ _, err = client.Get(context.Background(), &propertyv1.GetRequest{Metadata: md})
Expect(err).To(MatchError("rpc error: code = NotFound desc = banyandb: resource not found"))
})
})
diff --git a/ui/embed.go b/ui/embed.go
index 7d3657c..710e735 100644
--- a/ui/embed.go
+++ b/ui/embed.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package ui provides Web based UI for BanyanDB
+// Package ui provides Web based UI for BanyanDB.
package ui
import "embed"
[skywalking-banyandb] 03/03: Fix issues found by revive
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch lint
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit e8d655afa7c94fe2ad315962f58e2378a7df69d5
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Dec 5 01:03:26 2022 +0000
Fix issues found by revive
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
.golangci.yml | 1 +
api/common/id.go | 19 +-
api/data/measure.go | 8 +
api/data/stream.go | 5 +
api/event/measure.go | 6 +
api/event/stream.go | 6 +
banyand/kv/kv.go | 2 +-
banyand/liaison/grpc/registry_test.go | 2 +-
banyand/measure/measure.go | 2 +-
banyand/measure/measure_suite_test.go | 2 +-
banyand/metadata/metadata_test.go | 9 +-
banyand/metadata/schema/error.go | 17 +-
banyand/metadata/schema/etcd.go | 38 +--
banyand/metadata/schema/group.go | 12 +-
banyand/metadata/schema/index.go | 16 +-
banyand/metadata/schema/measure.go | 30 +--
banyand/metadata/schema/property.go | 8 +-
banyand/metadata/schema/schema.go | 30 ++-
banyand/metadata/schema/stream.go | 6 +-
banyand/metadata/schema/topn.go | 6 +-
banyand/stream/stream_suite_test.go | 2 +-
banyand/tsdb/index/writer.go | 6 +-
bydbctl/internal/cmd/group_test.go | 2 +-
bydbctl/internal/cmd/index_rule_binding_test.go | 2 +-
bydbctl/internal/cmd/index_rule_test.go | 2 +-
bydbctl/internal/cmd/measure_test.go | 4 +-
bydbctl/internal/cmd/property_test.go | 2 +-
bydbctl/internal/cmd/stream_test.go | 4 +-
docs/api-reference.md | 4 +-
go.mod | 2 +-
pkg/{convert/hash.go => bit/doc.go} | 13 +-
pkg/buffer/writer.go | 12 +
pkg/bus/bus.go | 84 ++++---
pkg/bus/bus_test.go | 2 +-
pkg/config/config.go | 7 +-
pkg/convert/{hash.go => doc.go} | 11 +-
pkg/convert/hash.go | 2 +
pkg/convert/number.go | 48 +---
pkg/convert/number_test.go | 3 +-
pkg/encoding/encoding.go | 5 +-
pkg/encoding/int.go | 5 +-
pkg/encoding/plain.go | 14 +-
pkg/flow/dedup_priority_queue.go | 3 +
pkg/flow/streaming/sink/slice.go | 82 -------
.../{flow_sliding_window.go => sliding_window.go} | 61 ++---
...iding_window_test.go => sliding_window_test.go} | 9 +-
pkg/flow/streaming/sources/channel.go | 26 +-
pkg/flow/streaming/sources/channel_test.go | 2 +-
pkg/flow/streaming/{flow.go => streaming.go} | 2 +
.../streaming/{flow_test.go => streaming_test.go} | 69 +++++-
pkg/flow/streaming/{flow_topn.go => topn.go} | 23 +-
.../streaming/{flow_topn_test.go => topn_test.go} | 0
pkg/flow/streaming/{flow_unary.go => unary.go} | 2 +-
pkg/flow/types.go | 11 +
pkg/flow/utils.go | 1 +
pkg/grpchelper/client.go | 5 +
pkg/index/index.go | 35 ++-
pkg/index/inverted/inverted.go | 11 +-
pkg/index/{ => lsm}/iterator.go | 138 ++---------
pkg/index/lsm/lsm.go | 3 +
pkg/index/lsm/search.go | 10 +-
pkg/index/posting/posting.go | 3 +
pkg/index/posting/roaring/roaring.go | 19 +-
pkg/index/testcases/duration.go | 8 +-
pkg/index/testcases/service_name.go | 6 +-
pkg/iter/iter.go | 1 +
pkg/logger/jsonproto.go | 2 +
pkg/logger/logger.go | 24 +-
pkg/partition/entity.go | 9 +
pkg/partition/index.go | 2 +
pkg/partition/route.go | 1 +
pkg/pb/v1/metadata.go | 7 +-
pkg/pb/v1/tsdb/interval.go | 6 +-
pkg/pb/v1/write.go | 266 +++------------------
pkg/query/aggregation/aggregation.go | 7 +-
pkg/query/executor/interface.go | 23 +-
pkg/query/logical/common.go | 38 +--
pkg/{convert/hash.go => query/logical/doc.go} | 15 +-
pkg/query/logical/expr.go | 35 +--
pkg/query/logical/expr_literal.go | 28 +--
pkg/query/logical/format.go | 21 +-
pkg/query/logical/index_filter.go | 70 +++---
pkg/query/logical/interface.go | 11 +-
pkg/query/logical/iter.go | 11 +-
pkg/query/logical/measure/measure_analyzer.go | 14 +-
pkg/query/logical/measure/measure_plan.go | 2 +-
.../logical/measure/measure_plan_aggregation.go | 4 +-
pkg/query/logical/measure/measure_plan_groupby.go | 3 +-
.../measure/measure_plan_indexscan_local.go | 25 +-
pkg/query/logical/measure/measure_plan_top.go | 16 +-
pkg/query/logical/measure/measure_top.go | 12 +
pkg/query/logical/measure/schema.go | 5 +-
pkg/query/logical/plan.go | 39 +--
pkg/query/logical/plan_orderby.go | 81 +------
pkg/query/logical/schema.go | 16 +-
pkg/query/logical/stream/schema.go | 3 +-
pkg/query/logical/stream/stream_analyzer.go | 6 +-
.../logical/stream/stream_plan_indexscan_local.go | 3 +-
pkg/query/logical/stream/stream_plan_tag_filter.go | 19 +-
pkg/query/logical/tag_filter.go | 49 ++--
pkg/run/run.go | 5 +
pkg/run/test.go | 18 +-
pkg/schema/metadata.go | 14 ++
pkg/signal/handler.go | 2 +
pkg/{convert/hash.go => test/doc.go} | 15 +-
pkg/test/flags/flags.go | 8 +-
pkg/test/flow/slice.go | 4 +-
pkg/test/helpers/context.go | 4 +
pkg/{convert/hash.go => test/helpers/doc.go} | 15 +-
pkg/test/helpers/fail_interceptor.go | 52 ----
pkg/test/helpers/grpc_health.go | 6 +-
pkg/test/helpers/http_health.go | 4 +-
pkg/test/matcher.go | 3 +
pkg/test/measure/etcd.go | 88 +------
pkg/test/measure/traffic/traffic.go | 187 ---------------
pkg/test/network.go | 1 +
pkg/test/setup.go | 7 +-
pkg/test/setup/setup.go | 6 +-
pkg/test/space.go | 4 +
pkg/test/stream/etcd.go | 93 +------
pkg/test/stream/traffic/searchable_template.json | 12 -
pkg/test/stream/traffic/traffic.go | 195 ---------------
pkg/timestamp/clock.go | 4 +-
pkg/{convert/hash.go => timestamp/doc.go} | 13 +-
pkg/timestamp/nano.go | 28 ++-
pkg/timestamp/range.go | 12 +
pkg/timestamp/scheduler.go | 5 +-
pkg/version/version.go | 2 +-
scripts/build/lint.mk | 2 +-
test/cases/stream/stream.go | 2 +-
test/integration/cold_query/query_suite_test.go | 2 +-
test/integration/load/load_suite_test.go | 2 +-
test/integration/other/measure_test.go | 2 +-
test/integration/other/property_test.go | 4 +-
test/integration/other/tls_test.go | 2 +-
test/integration/query/query_suite_test.go | 2 +-
136 files changed, 947 insertions(+), 1742 deletions(-)
diff --git a/.golangci.yml b/.golangci.yml
index 5094850..ae0cde1 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -141,6 +141,7 @@ linters-settings:
scope: toplevel
staticcheck:
go: "1.19"
+ checks: ["all", "-ST1000", "-ST1016", "-ST1020", "-ST1021", "-ST1022"]
stylecheck:
go: "1.19"
exhaustive:
diff --git a/api/common/id.go b/api/common/id.go
index ff0ed50..3a14c46 100644
--- a/api/common/id.go
+++ b/api/common/id.go
@@ -27,20 +27,28 @@ import (
)
type (
- // SeriesID identit fdf.
+ // SeriesID identity a series in a shard.
SeriesID uint64
- ShardID uint32
- ItemID uint64
+
+ // ShardID identity a shard in a tsdb.
+ ShardID uint32
+
+ // ItemID identity an item in a series.
+ ItemID uint64
)
+// Marshal encodes series id to bytes.
func (s SeriesID) Marshal() []byte {
return convert.Uint64ToBytes(uint64(s))
}
+// PositionKey is a context key to store the module position.
var PositionKey = contextPositionKey{}
type contextPositionKey struct{}
+// Position is stored in the context.
+// The logger could attach it for debugging.
type Position struct {
Module string
Database string
@@ -50,6 +58,7 @@ type Position struct {
KV string
}
+// Labels converts Position to Prom Labels.
func (p Position) Labels() prometheus.Labels {
return prometheus.Labels{
"module": p.Module,
@@ -61,6 +70,7 @@ func (p Position) Labels() prometheus.Labels {
}
}
+// SetPosition sets a position returned from fn to attach it to ctx, then return a new context.
func SetPosition(ctx context.Context, fn func(p Position) Position) context.Context {
val := ctx.Value(PositionKey)
var p Position
@@ -72,14 +82,17 @@ func SetPosition(ctx context.Context, fn func(p Position) Position) context.Cont
return context.WithValue(ctx, PositionKey, fn(p))
}
+// Error wraps a error msg.
type Error struct {
msg string
}
+// NewError returns a new Error.
func NewError(tpl string, args ...any) Error {
return Error{msg: fmt.Sprintf(tpl, args...)}
}
+// Msg shows the string msg.
func (e Error) Msg() string {
return e.msg
}
diff --git a/api/data/measure.go b/api/data/measure.go
index 08b3f1b..f2a8e41 100644
--- a/api/data/measure.go
+++ b/api/data/measure.go
@@ -22,21 +22,29 @@ import (
"github.com/apache/skywalking-banyandb/pkg/bus"
)
+// MeasureWriteKindVersion is the version tag of measure write kind.
var MeasureWriteKindVersion = common.KindVersion{
Version: "v1",
Kind: "measure-write",
}
+// TopicMeasureWrite is the measure write topic.
var TopicMeasureWrite = bus.UniTopic(MeasureWriteKindVersion.String())
+// MeasureQueryKindVersion is the version tag of measure query kind.
var MeasureQueryKindVersion = common.KindVersion{
Version: "v1",
Kind: "measure-query",
}
+
+// TopicMeasureQuery is the measure query topic.
var TopicMeasureQuery = bus.BiTopic(MeasureQueryKindVersion.String())
+// TopNQueryKindVersion is the version tag of top-n query kind.
var TopNQueryKindVersion = common.KindVersion{
Version: "v1",
Kind: "topN-query",
}
+
+// TopicTopNQuery is the top-n query topic.
var TopicTopNQuery = bus.BiTopic(TopNQueryKindVersion.String())
diff --git a/api/data/stream.go b/api/data/stream.go
index 1bc4a77..e083e7d 100644
--- a/api/data/stream.go
+++ b/api/data/stream.go
@@ -22,15 +22,20 @@ import (
"github.com/apache/skywalking-banyandb/pkg/bus"
)
+// StreamWriteKindVersion is the version tag of stream write kind.
var StreamWriteKindVersion = common.KindVersion{
Version: "v1",
Kind: "stream-write",
}
+// TopicStreamWrite is the stream write topic.
var TopicStreamWrite = bus.UniTopic(StreamWriteKindVersion.String())
+// StreamQueryKindVersion is the version tag of stream query kind.
var StreamQueryKindVersion = common.KindVersion{
Version: "v1",
Kind: "stream-query",
}
+
+// TopicStreamQuery is the stream query topic.
var TopicStreamQuery = bus.BiTopic(StreamQueryKindVersion.String())
diff --git a/api/event/measure.go b/api/event/measure.go
index 9f2524e..68516d6 100644
--- a/api/event/measure.go
+++ b/api/event/measure.go
@@ -23,15 +23,21 @@ import (
)
var (
+ // MeasureShardEventKindVersion is the version tag of measure shard event kind.
MeasureShardEventKindVersion = common.KindVersion{
Version: "v1",
Kind: "measure-event-shard",
}
+
+ // MeasureTopicShardEvent is the measure shard event publishing topic.
MeasureTopicShardEvent = bus.UniTopic(MeasureShardEventKindVersion.String())
+ // MeasureEntityEventKindVersion is the version tag of measure entity kind.
MeasureEntityEventKindVersion = common.KindVersion{
Version: "v1",
Kind: "measure-event-entity",
}
+
+ // MeasureTopicEntityEvent is the measure entity event publishing topic.
MeasureTopicEntityEvent = bus.UniTopic(MeasureEntityEventKindVersion.String())
)
diff --git a/api/event/stream.go b/api/event/stream.go
index fa6d39c..e39f01f 100644
--- a/api/event/stream.go
+++ b/api/event/stream.go
@@ -23,15 +23,21 @@ import (
)
var (
+ // StreamShardEventKindVersion is the version tag of stream shard entity kind.
StreamShardEventKindVersion = common.KindVersion{
Version: "v1",
Kind: "stream-event-shard",
}
+
+ // StreamTopicShardEvent is the stream entity event publishing topic.
StreamTopicShardEvent = bus.UniTopic(StreamShardEventKindVersion.String())
+ // StreamEntityEventKindVersion is the version tag of stream entity kind.
StreamEntityEventKindVersion = common.KindVersion{
Version: "v1",
Kind: "stream-event-entity",
}
+
+ // StreamTopicEntityEvent is the stream entity event publishing topic.
StreamTopicEntityEvent = bus.UniTopic(StreamEntityEventKindVersion.String())
)
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
index fb3960a..e8a3d78 100644
--- a/banyand/kv/kv.go
+++ b/banyand/kv/kv.go
@@ -140,7 +140,7 @@ func TSSWithMemTableSize(sizeInBytes int64) TimeSeriesOptions {
}
// Iterator allows iterating the kv tables.
-// TODO: use generic to provide a unique iterator
+// TODO: use generic to provide a unique iterator.
type Iterator interface {
Next()
Rewind()
diff --git a/banyand/liaison/grpc/registry_test.go b/banyand/liaison/grpc/registry_test.go
index d1cc679..b4feea7 100644
--- a/banyand/liaison/grpc/registry_test.go
+++ b/banyand/liaison/grpc/registry_test.go
@@ -185,7 +185,7 @@ func setupForRegistry() func() {
Expect(err).NotTo(HaveOccurred())
flags = append(flags, "--metadata-root-path="+metaPath, "--etcd-listen-client-url="+listenClientURL,
"--etcd-listen-peer-url="+listenPeerURL)
- deferFunc := test.SetUpModules(
+ deferFunc := test.SetupModules(
flags,
repo,
pipeline,
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 4a2e3be..2099859 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package measures implements a time-series-based storage which is consists of a sequence of data points.
+// Package measure implements a time-series-based storage which is consists of a sequence of data points.
// Each data point contains tags and fields. They arrive in a fixed interval. A data point could be updated
// by one with the identical entity(series_id) and timestamp.
package measure
diff --git a/banyand/measure/measure_suite_test.go b/banyand/measure/measure_suite_test.go
index d1711b9..8e33370 100644
--- a/banyand/measure/measure_suite_test.go
+++ b/banyand/measure/measure_suite_test.go
@@ -100,7 +100,7 @@ func setUp() (*services, func()) {
listenClientURL, listenPeerURL, err := test.NewEtcdListenUrls()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
flags = append(flags, "--etcd-listen-client-url="+listenClientURL, "--etcd-listen-peer-url="+listenPeerURL)
- moduleDeferFunc := test.SetUpModules(
+ moduleDeferFunc := test.SetupModules(
flags,
repo,
pipeline,
diff --git a/banyand/metadata/metadata_test.go b/banyand/metadata/metadata_test.go
index ebdf335..156480d 100644
--- a/banyand/metadata/metadata_test.go
+++ b/banyand/metadata/metadata_test.go
@@ -19,7 +19,6 @@ package metadata
import (
"context"
- "os"
"testing"
"github.com/stretchr/testify/assert"
@@ -27,6 +26,7 @@ import (
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/pkg/logger"
+ testhelper "github.com/apache/skywalking-banyandb/pkg/test"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
test "github.com/apache/skywalking-banyandb/pkg/test/stream"
)
@@ -43,14 +43,15 @@ func Test_service_RulesBySubject(t *testing.T) {
ctx := context.TODO()
s, _ := NewService(ctx)
is.NotNil(s)
- rootDir := test.RandomTempDir()
- err := s.FlagSet().Parse([]string{"--metadata-root-path=" + rootDir})
+ rootDir, deferFn, err := testhelper.NewSpace()
+ is.NoError(err)
+ err = s.FlagSet().Parse([]string{"--metadata-root-path=" + rootDir})
is.NoError(err)
err = s.PreRun()
is.NoError(err)
defer func() {
s.GracefulStop()
- _ = os.RemoveAll(rootDir)
+ deferFn()
}()
err = test.PreloadSchema(s.SchemaRegistry())
diff --git a/banyand/metadata/schema/error.go b/banyand/metadata/schema/error.go
index d9cf005..ee4efa6 100644
--- a/banyand/metadata/schema/error.go
+++ b/banyand/metadata/schema/error.go
@@ -25,20 +25,17 @@ import (
)
var (
+ // ErrGRPCResourceNotFound indicates the resource doesn't exist.
+ ErrGRPCResourceNotFound = statusGRPCResourceNotFound.Err()
+
statusGRPCInvalidArgument = status.New(codes.InvalidArgument, "banyandb: input is invalid")
- ErrGRPCInvalidArgument = statusGRPCInvalidArgument.Err()
statusGRPCResourceNotFound = status.New(codes.NotFound, "banyandb: resource not found")
- ErrGRPCResourceNotFound = statusGRPCResourceNotFound.Err()
statusGRPCAlreadyExists = status.New(codes.AlreadyExists, "banyandb: resource already exists")
- ErrGRPCAlreadyExists = statusGRPCAlreadyExists.Err()
+ errGRPCAlreadyExists = statusGRPCAlreadyExists.Err()
statusDataLoss = status.New(codes.DataLoss, "banyandb: resource corrupts.")
- ErrGRPCDataLoss = statusDataLoss.Err()
+ errGRPCDataLoss = statusDataLoss.Err()
)
-func IsNotFound(err error) bool {
- return errors.Is(err, ErrGRPCResourceNotFound)
-}
-
// BadRequest creates a gRPC error with error details with type BadRequest,
// which describes violations in a client request.
func BadRequest(field, desc string) error {
@@ -51,3 +48,7 @@ func BadRequest(field, desc string) error {
st, _ := statusGRPCInvalidArgument.WithDetails(br)
return st.Err()
}
+
+func isNotFound(err error) bool {
+ return errors.Is(err, ErrGRPCResourceNotFound)
+}
diff --git a/banyand/metadata/schema/etcd.go b/banyand/metadata/schema/etcd.go
index f3b3fa9..d7c4cf3 100644
--- a/banyand/metadata/schema/etcd.go
+++ b/banyand/metadata/schema/etcd.go
@@ -42,29 +42,34 @@ var (
_ Group = (*etcdSchemaRegistry)(nil)
_ Property = (*etcdSchemaRegistry)(nil)
- ErrUnexpectedNumberOfEntities = errors.New("unexpected number of entities")
- ErrConcurrentModification = errors.New("concurrent modification of entities")
+ errUnexpectedNumberOfEntities = errors.New("unexpected number of entities")
+ errConcurrentModification = errors.New("concurrent modification of entities")
)
+// HasMetadata allows getting Metadata.
type HasMetadata interface {
GetMetadata() *commonv1.Metadata
proto.Message
}
+// RegistryOption is the option to create Registry.
type RegistryOption func(*etcdSchemaRegistryConfig)
+// RootDir sets the root directory of Registry.
func RootDir(rootDir string) RegistryOption {
return func(config *etcdSchemaRegistryConfig) {
config.rootDir = rootDir
}
}
+// LoggerLevel sets the logger level.
func LoggerLevel(loggerLevel string) RegistryOption {
return func(config *etcdSchemaRegistryConfig) {
config.loggerLevel = loggerLevel
}
}
+// ConfigureListener sets client and peer urls of listeners.
func ConfigureListener(lc, lp string) RegistryOption {
return func(config *etcdSchemaRegistryConfig) {
config.listenerClientURL = lc
@@ -77,7 +82,7 @@ type eventHandler struct {
interestKeys Kind
}
-func (eh *eventHandler) InterestOf(kind Kind) bool {
+func (eh *eventHandler) interestOf(kind Kind) bool {
return KindMask&kind&eh.interestKeys != 0
}
@@ -114,7 +119,7 @@ func (e *etcdSchemaRegistry) notifyUpdate(metadata Metadata) {
hh := e.handlers
e.mux.RUnlock()
for _, h := range hh {
- if h.InterestOf(metadata.Kind) {
+ if h.interestOf(metadata.Kind) {
h.handler.OnAddOrUpdate(metadata)
}
}
@@ -125,7 +130,7 @@ func (e *etcdSchemaRegistry) notifyDelete(metadata Metadata) {
hh := e.handlers
e.mux.RUnlock()
for _, h := range hh {
- if h.InterestOf(metadata.Kind) {
+ if h.interestOf(metadata.Kind) {
h.handler.OnDelete(metadata)
}
}
@@ -149,6 +154,7 @@ func (e *etcdSchemaRegistry) Close() error {
return nil
}
+// NewEtcdSchemaRegistry returns a Registry powered by Etcd.
func NewEtcdSchemaRegistry(options ...RegistryOption) (Registry, error) {
registryConfig := &etcdSchemaRegistryConfig{
rootDir: os.TempDir(),
@@ -189,7 +195,7 @@ func (e *etcdSchemaRegistry) get(ctx context.Context, key string, message proto.
return ErrGRPCResourceNotFound
}
if resp.Count > 1 {
- return ErrUnexpectedNumberOfEntities
+ return errUnexpectedNumberOfEntities
}
if err = proto.Unmarshal(resp.Kvs[0].Value, message); err != nil {
return err
@@ -206,7 +212,7 @@ func (e *etcdSchemaRegistry) get(ctx context.Context, key string, message proto.
// and overwrite the existing value if so.
// Otherwise, it will return ErrGRPCResourceNotFound.
func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) error {
- key, err := metadata.Key()
+ key, err := metadata.key()
if err != nil {
return err
}
@@ -215,7 +221,7 @@ func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) erro
return err
}
if getResp.Count > 1 {
- return ErrUnexpectedNumberOfEntities
+ return errUnexpectedNumberOfEntities
}
val, err := proto.Marshal(metadata.Spec.(proto.Message))
if err != nil {
@@ -228,7 +234,7 @@ func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) erro
return innerErr
}
// directly return if we have the same entity
- if metadata.Equal(existingVal) {
+ if metadata.equal(existingVal) {
return nil
}
@@ -241,7 +247,7 @@ func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) erro
return txnErr
}
if !txnResp.Succeeded {
- return ErrConcurrentModification
+ return errConcurrentModification
}
} else {
return ErrGRPCResourceNotFound
@@ -254,7 +260,7 @@ func (e *etcdSchemaRegistry) update(ctx context.Context, metadata Metadata) erro
// and put the value if it does not exist.
// Otherwise, it will return ErrGRPCAlreadyExists.
func (e *etcdSchemaRegistry) create(ctx context.Context, metadata Metadata) error {
- key, err := metadata.Key()
+ key, err := metadata.key()
if err != nil {
return err
}
@@ -263,7 +269,7 @@ func (e *etcdSchemaRegistry) create(ctx context.Context, metadata Metadata) erro
return err
}
if getResp.Count > 1 {
- return ErrUnexpectedNumberOfEntities
+ return errUnexpectedNumberOfEntities
}
val, err := proto.Marshal(metadata.Spec.(proto.Message))
if err != nil {
@@ -271,7 +277,7 @@ func (e *etcdSchemaRegistry) create(ctx context.Context, metadata Metadata) erro
}
replace := getResp.Count > 0
if replace {
- return ErrGRPCAlreadyExists
+ return errGRPCAlreadyExists
}
_, err = e.kv.Put(ctx, key, string(val))
if err != nil {
@@ -304,11 +310,11 @@ func (e *etcdSchemaRegistry) listWithPrefix(ctx context.Context, prefix string,
}
func listPrefixesForEntity(group, entityPrefix string) string {
- return GroupsKeyPrefix + group + entityPrefix
+ return groupsKeyPrefix + group + entityPrefix
}
func (e *etcdSchemaRegistry) delete(ctx context.Context, metadata Metadata) (bool, error) {
- key, err := metadata.Key()
+ key, err := metadata.key()
if err != nil {
return false, err
}
@@ -350,7 +356,7 @@ func (e *etcdSchemaRegistry) delete(ctx context.Context, metadata Metadata) (boo
}
func formatKey(entityPrefix string, metadata *commonv1.Metadata) string {
- return GroupsKeyPrefix + metadata.GetGroup() + entityPrefix + metadata.GetName()
+ return groupsKeyPrefix + metadata.GetGroup() + entityPrefix + metadata.GetName()
}
func incrementLastByte(key string) string {
diff --git a/banyand/metadata/schema/group.go b/banyand/metadata/schema/group.go
index b62742b..3ecb0c7 100644
--- a/banyand/metadata/schema/group.go
+++ b/banyand/metadata/schema/group.go
@@ -29,8 +29,8 @@ import (
)
var (
- GroupsKeyPrefix = "/groups/"
- GroupMetadataKey = "/__meta_group__"
+ groupsKeyPrefix = "/groups/"
+ groupMetadataKey = "/__meta_group__"
)
func (e *etcdSchemaRegistry) GetGroup(ctx context.Context, group string) (*commonv1.Group, error) {
@@ -43,7 +43,7 @@ func (e *etcdSchemaRegistry) GetGroup(ctx context.Context, group string) (*commo
}
func (e *etcdSchemaRegistry) ListGroup(ctx context.Context) ([]*commonv1.Group, error) {
- messages, err := e.kv.Get(ctx, GroupsKeyPrefix, clientv3.WithFromKey(), clientv3.WithRange(incrementLastByte(GroupsKeyPrefix)))
+ messages, err := e.kv.Get(ctx, groupsKeyPrefix, clientv3.WithFromKey(), clientv3.WithRange(incrementLastByte(groupsKeyPrefix)))
if err != nil {
return nil, err
}
@@ -51,7 +51,7 @@ func (e *etcdSchemaRegistry) ListGroup(ctx context.Context) ([]*commonv1.Group,
var groups []*commonv1.Group
for _, kv := range messages.Kvs {
// kv.Key = "/groups/" + {group} + "/__meta_info__"
- if strings.HasSuffix(string(kv.Key), GroupMetadataKey) {
+ if strings.HasSuffix(string(kv.Key), groupMetadataKey) {
message := &commonv1.Group{}
if innerErr := proto.Unmarshal(kv.Value, message); innerErr != nil {
return nil, innerErr
@@ -68,7 +68,7 @@ func (e *etcdSchemaRegistry) DeleteGroup(ctx context.Context, group string) (boo
if err != nil {
return false, errors.Wrap(err, group)
}
- keyPrefix := GroupsKeyPrefix + g.GetMetadata().GetName() + "/"
+ keyPrefix := groupsKeyPrefix + g.GetMetadata().GetName() + "/"
resp, err := e.kv.Delete(ctx, keyPrefix, clientv3.WithRange(incrementLastByte(keyPrefix)))
if err != nil {
return false, err
@@ -107,5 +107,5 @@ func (e *etcdSchemaRegistry) UpdateGroup(ctx context.Context, group *commonv1.Gr
}
func formatGroupKey(group string) string {
- return GroupsKeyPrefix + group + GroupMetadataKey
+ return groupsKeyPrefix + group + groupMetadataKey
}
diff --git a/banyand/metadata/schema/index.go b/banyand/metadata/schema/index.go
index f3bf415..d00166e 100644
--- a/banyand/metadata/schema/index.go
+++ b/banyand/metadata/schema/index.go
@@ -28,8 +28,8 @@ import (
)
var (
- IndexRuleBindingKeyPrefix = "/index-rule-bindings/"
- IndexRuleKeyPrefix = "/index-rules/"
+ indexRuleBindingKeyPrefix = "/index-rule-bindings/"
+ indexRuleKeyPrefix = "/index-rules/"
)
func (e *etcdSchemaRegistry) GetIndexRuleBinding(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.IndexRuleBinding, error) {
@@ -44,7 +44,7 @@ func (e *etcdSchemaRegistry) ListIndexRuleBinding(ctx context.Context, opt ListO
if opt.Group == "" {
return nil, BadRequest("group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, IndexRuleBindingKeyPrefix), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, indexRuleBindingKeyPrefix), func() proto.Message {
return &databasev1.IndexRuleBinding{}
})
if err != nil {
@@ -95,7 +95,7 @@ func (e *etcdSchemaRegistry) GetIndexRule(ctx context.Context, metadata *commonv
return nil, err
}
if entity.Metadata.Id == 0 {
- return nil, ErrGRPCDataLoss
+ return nil, errGRPCDataLoss
}
return &entity, nil
}
@@ -104,7 +104,7 @@ func (e *etcdSchemaRegistry) ListIndexRule(ctx context.Context, opt ListOpt) ([]
if opt.Group == "" {
return nil, BadRequest("group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, IndexRuleKeyPrefix), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, indexRuleKeyPrefix), func() proto.Message {
return &databasev1.IndexRule{}
})
if err != nil {
@@ -114,7 +114,7 @@ func (e *etcdSchemaRegistry) ListIndexRule(ctx context.Context, opt ListOpt) ([]
for _, message := range messages {
entity := message.(*databasev1.IndexRule)
if entity.Metadata.Id == 0 {
- return nil, ErrGRPCDataLoss
+ return nil, errGRPCDataLoss
}
entities = append(entities, entity)
}
@@ -159,9 +159,9 @@ func (e *etcdSchemaRegistry) DeleteIndexRule(ctx context.Context, metadata *comm
}
func formatIndexRuleKey(metadata *commonv1.Metadata) string {
- return formatKey(IndexRuleKeyPrefix, metadata)
+ return formatKey(indexRuleKeyPrefix, metadata)
}
func formatIndexRuleBindingKey(metadata *commonv1.Metadata) string {
- return formatKey(IndexRuleBindingKeyPrefix, metadata)
+ return formatKey(indexRuleBindingKeyPrefix, metadata)
}
diff --git a/banyand/metadata/schema/measure.go b/banyand/metadata/schema/measure.go
index 7bb71c5..8895b8b 100644
--- a/banyand/metadata/schema/measure.go
+++ b/banyand/metadata/schema/measure.go
@@ -29,8 +29,8 @@ import (
)
const (
- MeasureKeyPrefix = "/measures/"
- TagTypeID = "id"
+ measureKeyPrefix = "/measures/"
+ tagTypeID = "id"
)
func (e *etcdSchemaRegistry) GetMeasure(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Measure, error) {
@@ -45,7 +45,7 @@ func (e *etcdSchemaRegistry) ListMeasure(ctx context.Context, opt ListOpt) ([]*d
if opt.Group == "" {
return nil, BadRequest("group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, MeasureKeyPrefix), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, measureKeyPrefix), func() proto.Message {
return &databasev1.Measure{}
})
if err != nil {
@@ -72,14 +72,14 @@ func (e *etcdSchemaRegistry) CreateMeasure(ctx context.Context, measure *databas
// Add an index rule for the ID type tag
idIndexRuleMetadata := &commonv1.Metadata{
- Name: TagTypeID,
+ Name: tagTypeID,
Group: measure.Metadata.Group,
}
_, err := e.GetIndexRule(ctx, idIndexRuleMetadata)
- if IsNotFound(err) {
+ if isNotFound(err) {
if errIndexRule := e.CreateIndexRule(ctx, &databasev1.IndexRule{
Metadata: idIndexRuleMetadata,
- Tags: []string{TagTypeID},
+ Tags: []string{tagTypeID},
Type: databasev1.IndexRule_TYPE_TREE,
Location: databasev1.IndexRule_LOCATION_SERIES,
UpdatedAt: timestamppb.Now(),
@@ -99,10 +99,10 @@ func (e *etcdSchemaRegistry) CreateMeasure(ctx context.Context, measure *databas
}
irb := &databasev1.IndexRuleBinding{
Metadata: &commonv1.Metadata{
- Name: TagTypeID + "_" + measure.Metadata.Name + "_" + ts.Name,
+ Name: tagTypeID + "_" + measure.Metadata.Name + "_" + ts.Name,
Group: measure.Metadata.Group,
},
- Rules: []string{TagTypeID},
+ Rules: []string{tagTypeID},
Subject: &databasev1.Subject{
Catalog: commonv1.Catalog_CATALOG_MEASURE,
Name: measure.Metadata.Name,
@@ -115,7 +115,7 @@ func (e *etcdSchemaRegistry) CreateMeasure(ctx context.Context, measure *databas
if innerErr == nil {
return e.UpdateIndexRuleBinding(ctx, irb)
}
- if IsNotFound(innerErr) {
+ if isNotFound(innerErr) {
return e.CreateIndexRuleBinding(ctx, irb)
}
return innerErr
@@ -139,14 +139,14 @@ func (e *etcdSchemaRegistry) UpdateMeasure(ctx context.Context, measure *databas
// Add an index rule for the ID type tag
idIndexRuleMetadata := &commonv1.Metadata{
- Name: TagTypeID,
+ Name: tagTypeID,
Group: measure.Metadata.Group,
}
_, err := e.GetIndexRule(ctx, idIndexRuleMetadata)
- if IsNotFound(err) {
+ if isNotFound(err) {
if errIndexRule := e.CreateIndexRule(ctx, &databasev1.IndexRule{
Metadata: idIndexRuleMetadata,
- Tags: []string{TagTypeID},
+ Tags: []string{tagTypeID},
Type: databasev1.IndexRule_TYPE_TREE,
Location: databasev1.IndexRule_LOCATION_SERIES,
UpdatedAt: timestamppb.Now(),
@@ -166,10 +166,10 @@ func (e *etcdSchemaRegistry) UpdateMeasure(ctx context.Context, measure *databas
}
if errIndexRule := e.UpdateIndexRuleBinding(ctx, &databasev1.IndexRuleBinding{
Metadata: &commonv1.Metadata{
- Name: TagTypeID + "_" + measure.Metadata.Name + "_" + ts.Name,
+ Name: tagTypeID + "_" + measure.Metadata.Name + "_" + ts.Name,
Group: measure.Metadata.Group,
},
- Rules: []string{TagTypeID},
+ Rules: []string{tagTypeID},
Subject: &databasev1.Subject{
Catalog: commonv1.Catalog_CATALOG_MEASURE,
Name: measure.Metadata.Name,
@@ -214,5 +214,5 @@ func (e *etcdSchemaRegistry) TopNAggregations(ctx context.Context, metadata *com
}
func formatMeasureKey(metadata *commonv1.Metadata) string {
- return formatKey(MeasureKeyPrefix, metadata)
+ return formatKey(measureKeyPrefix, metadata)
}
diff --git a/banyand/metadata/schema/property.go b/banyand/metadata/schema/property.go
index e8c79ff..b4e8024 100644
--- a/banyand/metadata/schema/property.go
+++ b/banyand/metadata/schema/property.go
@@ -29,7 +29,7 @@ import (
const all = "*"
-var PropertyKeyPrefix = "/properties/"
+var propertyKeyPrefix = "/properties/"
func (e *etcdSchemaRegistry) GetProperty(ctx context.Context, metadata *propertyv1.Metadata, tags []string) (*propertyv1.Property, error) {
var entity propertyv1.Property
@@ -62,7 +62,7 @@ func (e *etcdSchemaRegistry) ListProperty(ctx context.Context, container *common
if container.Group == "" {
return nil, BadRequest("container.group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(container.Group, PropertyKeyPrefix+container.Name), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(container.Group, propertyKeyPrefix+container.Name), func() proto.Message {
return &propertyv1.Property{}
})
if err != nil {
@@ -99,7 +99,7 @@ func (e *etcdSchemaRegistry) ApplyProperty(ctx context.Context, property *proper
if err == nil {
return true, tagsNum, nil
}
- if !errors.Is(err, ErrGRPCAlreadyExists) {
+ if !errors.Is(err, errGRPCAlreadyExists) {
return false, 0, err
}
if strategy != propertyv1.ApplyRequest_STRATEGY_REPLACE {
@@ -165,5 +165,5 @@ func transformKey(metadata *propertyv1.Metadata) *commonv1.Metadata {
}
func formatPropertyKey(metadata *commonv1.Metadata) string {
- return formatKey(PropertyKeyPrefix, metadata)
+ return formatKey(propertyKeyPrefix, metadata)
}
diff --git a/banyand/metadata/schema/schema.go b/banyand/metadata/schema/schema.go
index 02aacd3..e26ad94 100644
--- a/banyand/metadata/schema/schema.go
+++ b/banyand/metadata/schema/schema.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package schema implements CRUD schema.
package schema
import (
@@ -29,15 +30,18 @@ import (
propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
)
-var ErrUnsupportedEntityType = errors.New("unsupported entity type")
+var errUnsupportedEntityType = errors.New("unsupported entity type")
+// Kind is the type of a resource.
type Kind int
+// EventHandler allows receiving and handling the resource change events.
type EventHandler interface {
OnAddOrUpdate(Metadata)
OnDelete(Metadata)
}
+// KindMask tends to check whether an event is valid.
const (
KindGroup Kind = 1 << iota
KindStream
@@ -46,14 +50,15 @@ const (
KindIndexRule
KindTopNAggregation
KindProperty
+ KindMask = KindGroup | KindStream | KindMeasure | KindIndexRuleBinding | KindIndexRule | KindTopNAggregation
)
-const KindMask = KindGroup | KindStream | KindMeasure | KindIndexRuleBinding | KindIndexRule | KindTopNAggregation
-
+// ListOpt contains options to list resources.
type ListOpt struct {
Group string
}
+// Registry allowing depositing resources.
type Registry interface {
io.Closer
ReadyNotify() <-chan struct{}
@@ -68,19 +73,23 @@ type Registry interface {
Property
}
+// TypeMeta defines the identity and type of an Event.
type TypeMeta struct {
Name string
Group string
Kind Kind
}
+// Metadata wrap dedicated serialized resource and its TypeMeta.
type Metadata struct {
Spec Spec
TypeMeta
}
+// Spec is a placeholder of a serialized resource.
type Spec interface{}
+// Unmarshal encode bytes to proto.Message.
func (tm TypeMeta) Unmarshal(data []byte) (m proto.Message, err error) {
switch tm.Kind {
case KindGroup:
@@ -98,13 +107,13 @@ func (tm TypeMeta) Unmarshal(data []byte) (m proto.Message, err error) {
case KindTopNAggregation:
m = &databasev1.TopNAggregation{}
default:
- return nil, ErrUnsupportedEntityType
+ return nil, errUnsupportedEntityType
}
err = proto.Unmarshal(data, m)
return
}
-func (m Metadata) Key() (string, error) {
+func (m Metadata) key() (string, error) {
switch m.Kind {
case KindGroup:
return formatGroupKey(m.Name), nil
@@ -139,11 +148,11 @@ func (m Metadata) Key() (string, error) {
Name: m.Name,
}), nil
default:
- return "", ErrUnsupportedEntityType
+ return "", errUnsupportedEntityType
}
}
-func (m Metadata) Equal(other proto.Message) bool {
+func (m Metadata) equal(other proto.Message) bool {
if other == nil {
return false
}
@@ -155,6 +164,7 @@ func (m Metadata) Equal(other proto.Message) bool {
return false
}
+// Stream allows CRUD stream schemas in a group.
type Stream interface {
GetStream(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Stream, error)
ListStream(ctx context.Context, opt ListOpt) ([]*databasev1.Stream, error)
@@ -164,6 +174,7 @@ type Stream interface {
RegisterHandler(Kind, EventHandler)
}
+// IndexRule allows CRUD index rule schemas in a group.
type IndexRule interface {
GetIndexRule(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.IndexRule, error)
ListIndexRule(ctx context.Context, opt ListOpt) ([]*databasev1.IndexRule, error)
@@ -172,6 +183,7 @@ type IndexRule interface {
DeleteIndexRule(ctx context.Context, metadata *commonv1.Metadata) (bool, error)
}
+// IndexRuleBinding allows CRUD index rule binding schemas in a group.
type IndexRuleBinding interface {
GetIndexRuleBinding(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.IndexRuleBinding, error)
ListIndexRuleBinding(ctx context.Context, opt ListOpt) ([]*databasev1.IndexRuleBinding, error)
@@ -180,6 +192,7 @@ type IndexRuleBinding interface {
DeleteIndexRuleBinding(ctx context.Context, metadata *commonv1.Metadata) (bool, error)
}
+// Measure allows CRUD measure schemas in a group.
type Measure interface {
GetMeasure(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Measure, error)
ListMeasure(ctx context.Context, opt ListOpt) ([]*databasev1.Measure, error)
@@ -190,6 +203,7 @@ type Measure interface {
TopNAggregations(ctx context.Context, metadata *commonv1.Metadata) ([]*databasev1.TopNAggregation, error)
}
+// Group allows CRUD groups which is namespaces of resources.
type Group interface {
GetGroup(ctx context.Context, group string) (*commonv1.Group, error)
ListGroup(ctx context.Context) ([]*commonv1.Group, error)
@@ -199,6 +213,7 @@ type Group interface {
UpdateGroup(ctx context.Context, group *commonv1.Group) error
}
+// TopNAggregation allows CRUD top-n aggregation schemas in a group.
type TopNAggregation interface {
GetTopNAggregation(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.TopNAggregation, error)
ListTopNAggregation(ctx context.Context, opt ListOpt) ([]*databasev1.TopNAggregation, error)
@@ -207,6 +222,7 @@ type TopNAggregation interface {
DeleteTopNAggregation(ctx context.Context, metadata *commonv1.Metadata) (bool, error)
}
+// Property allows CRUD properties or tags in group.
type Property interface {
GetProperty(ctx context.Context, metadata *propertyv1.Metadata, tags []string) (*propertyv1.Property, error)
ListProperty(ctx context.Context, container *commonv1.Metadata, ids []string, tags []string) ([]*propertyv1.Property, error)
diff --git a/banyand/metadata/schema/stream.go b/banyand/metadata/schema/stream.go
index 433f35e..57427e3 100644
--- a/banyand/metadata/schema/stream.go
+++ b/banyand/metadata/schema/stream.go
@@ -26,7 +26,7 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
)
-var StreamKeyPrefix = "/streams/"
+var streamKeyPrefix = "/streams/"
func (e *etcdSchemaRegistry) GetStream(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Stream, error) {
var entity databasev1.Stream
@@ -40,7 +40,7 @@ func (e *etcdSchemaRegistry) ListStream(ctx context.Context, opt ListOpt) ([]*da
if opt.Group == "" {
return nil, BadRequest("group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, StreamKeyPrefix), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, streamKeyPrefix), func() proto.Message {
return &databasev1.Stream{}
})
if err != nil {
@@ -91,5 +91,5 @@ func (e *etcdSchemaRegistry) DeleteStream(ctx context.Context, metadata *commonv
}
func formatStreamKey(metadata *commonv1.Metadata) string {
- return formatKey(StreamKeyPrefix, metadata)
+ return formatKey(streamKeyPrefix, metadata)
}
diff --git a/banyand/metadata/schema/topn.go b/banyand/metadata/schema/topn.go
index fc94dad..4ba89b9 100644
--- a/banyand/metadata/schema/topn.go
+++ b/banyand/metadata/schema/topn.go
@@ -26,7 +26,7 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
)
-var TopNAggregationKeyPrefix = "/topnagg/"
+var topNAggregationKeyPrefix = "/topnagg/"
func (e *etcdSchemaRegistry) GetTopNAggregation(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.TopNAggregation, error) {
var entity databasev1.TopNAggregation
@@ -40,7 +40,7 @@ func (e *etcdSchemaRegistry) ListTopNAggregation(ctx context.Context, opt ListOp
if opt.Group == "" {
return nil, BadRequest("group", "group should not be empty")
}
- messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, TopNAggregationKeyPrefix), func() proto.Message {
+ messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, topNAggregationKeyPrefix), func() proto.Message {
return &databasev1.TopNAggregation{}
})
if err != nil {
@@ -86,5 +86,5 @@ func (e *etcdSchemaRegistry) DeleteTopNAggregation(ctx context.Context, metadata
}
func formatTopNAggregationKey(metadata *commonv1.Metadata) string {
- return formatKey(TopNAggregationKeyPrefix, metadata)
+ return formatKey(topNAggregationKeyPrefix, metadata)
}
diff --git a/banyand/stream/stream_suite_test.go b/banyand/stream/stream_suite_test.go
index acc9955..d6375b1 100644
--- a/banyand/stream/stream_suite_test.go
+++ b/banyand/stream/stream_suite_test.go
@@ -99,7 +99,7 @@ func setUp() (*services, func()) {
listenClientURL, listenPeerURL, err := test.NewEtcdListenUrls()
Expect(err).NotTo(HaveOccurred())
flags = append(flags, "--etcd-listen-client-url="+listenClientURL, "--etcd-listen-peer-url="+listenPeerURL)
- moduleDeferFunc := test.SetUpModules(
+ moduleDeferFunc := test.SetupModules(
flags,
repo,
pipeline,
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
index 7385466..2958c8f 100644
--- a/banyand/tsdb/index/writer.go
+++ b/banyand/tsdb/index/writer.go
@@ -244,12 +244,12 @@ func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]
return nil, errors.WithMessagef(err, "index rule:%v", ruleIndex.Rule.Metadata)
}
fv, err := pbv1.ParseTagValue(tag)
- if errors.Is(err, pbv1.ErrNullValue) {
- return nil, nil
- }
if err != nil {
return nil, err
}
+ if fv.GetValue() == nil && fv.GetArr() == nil {
+ return nil, nil
+ }
v := fv.GetValue()
if v != nil {
val = append(val, v)
diff --git a/bydbctl/internal/cmd/group_test.go b/bydbctl/internal/cmd/group_test.go
index 5a84abc..6c18a8f 100644
--- a/bydbctl/internal/cmd/group_test.go
+++ b/bydbctl/internal/cmd/group_test.go
@@ -37,7 +37,7 @@ var _ = Describe("Group", func() {
var deferFunc func()
var rootCmd *cobra.Command
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating group
diff --git a/bydbctl/internal/cmd/index_rule_binding_test.go b/bydbctl/internal/cmd/index_rule_binding_test.go
index 99ac03d..7e92968 100644
--- a/bydbctl/internal/cmd/index_rule_binding_test.go
+++ b/bydbctl/internal/cmd/index_rule_binding_test.go
@@ -37,7 +37,7 @@ var _ = Describe("IndexRuleBindingSchema Operation", func() {
var deferFunc func()
var rootCmd *cobra.Command
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating indexRuleBinding schema
diff --git a/bydbctl/internal/cmd/index_rule_test.go b/bydbctl/internal/cmd/index_rule_test.go
index ede7edb..f159b5c 100644
--- a/bydbctl/internal/cmd/index_rule_test.go
+++ b/bydbctl/internal/cmd/index_rule_test.go
@@ -37,7 +37,7 @@ var _ = Describe("IndexRuleSchema Operation", func() {
var deferFunc func()
var rootCmd *cobra.Command
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating indexRule schema
diff --git a/bydbctl/internal/cmd/measure_test.go b/bydbctl/internal/cmd/measure_test.go
index 28c8334..cfb369b 100644
--- a/bydbctl/internal/cmd/measure_test.go
+++ b/bydbctl/internal/cmd/measure_test.go
@@ -44,7 +44,7 @@ var _ = Describe("Measure Schema Operation", func() {
var deferFunc func()
var rootCmd *cobra.Command
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating measure schema
@@ -182,7 +182,7 @@ var _ = Describe("Measure Data Query", func() {
startStr = now.Add(-20 * time.Minute).Format(time.RFC3339)
interval = 1 * time.Millisecond
endStr = now.Add(5 * time.Minute).Format(time.RFC3339)
- grpcAddr, addr, deferFunc = setup.SetUp()
+ grpcAddr, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
rootCmd = &cobra.Command{Use: "root"}
diff --git a/bydbctl/internal/cmd/property_test.go b/bydbctl/internal/cmd/property_test.go
index 95ac1f9..74c4c8d 100644
--- a/bydbctl/internal/cmd/property_test.go
+++ b/bydbctl/internal/cmd/property_test.go
@@ -75,7 +75,7 @@ tags:
p2Proto := new(propertyv1.Property)
helpers.UnmarshalYAML([]byte(p2YAML), p2Proto)
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating property schema
diff --git a/bydbctl/internal/cmd/stream_test.go b/bydbctl/internal/cmd/stream_test.go
index 1790952..24322f1 100644
--- a/bydbctl/internal/cmd/stream_test.go
+++ b/bydbctl/internal/cmd/stream_test.go
@@ -44,7 +44,7 @@ var _ = Describe("Stream Schema Operation", func() {
var deferFunc func()
var rootCmd *cobra.Command
BeforeEach(func() {
- _, addr, deferFunc = setup.SetUp()
+ _, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
// extracting the operation of creating stream schema
@@ -188,7 +188,7 @@ var _ = Describe("Stream Data Query", func() {
nowStr = now.Format(time.RFC3339)
interval = 500 * time.Millisecond
endStr = now.Add(1 * time.Hour).Format(time.RFC3339)
- grpcAddr, addr, deferFunc = setup.SetUp()
+ grpcAddr, addr, deferFunc = setup.Common()
Eventually(helpers.HTTPHealthCheck(addr), flags.EventuallyTimeout).Should(Succeed())
addr = "http://" + addr
rootCmd = &cobra.Command{Use: "root"}
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 799d11a..2c5839b 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -3,7 +3,7 @@
## Table of Contents
-- [banyandb/common/v1/common.proto](#banyandb_commonv1_common-proto)
+- [banyandb/common/v1/common.proto](#banyandb_common_v1_common-proto)
- [Group](#banyandb-common-v1-Group)
- [IntervalRule](#banyandb-common-v1-IntervalRule)
- [Metadata](#banyandb-common-v1-Metadata)
@@ -212,7 +212,7 @@
-<a name="banyandb_commonv1_common-proto"></a>
+<a name="banyandb_common_v1_common-proto"></a>
<p align="right"><a href="#top">Top</a></p>
## banyandb/common/v1/common.proto
diff --git a/go.mod b/go.mod
index d8191c3..d52b8c1 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,6 @@ require (
github.com/blugelabs/bluge v0.2.2
github.com/cespare/xxhash v1.1.0
github.com/dgraph-io/badger/v3 v3.2011.1
- github.com/dgraph-io/ristretto v0.1.0
github.com/emirpasic/gods v1.18.1
github.com/envoyproxy/protoc-gen-validate v0.1.0
github.com/go-chi/chi/v5 v5.0.7
@@ -60,6 +59,7 @@ require (
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
diff --git a/pkg/convert/hash.go b/pkg/bit/doc.go
similarity index 82%
copy from pkg/convert/hash.go
copy to pkg/bit/doc.go
index b08c2a5..18bf1f6 100644
--- a/pkg/convert/hash.go
+++ b/pkg/bit/doc.go
@@ -15,14 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package convert
+// TODO:// move this into package encoding.
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
+// Package bit implement binary packer and unpacker to build custom binary stream.
+package bit
diff --git a/pkg/buffer/writer.go b/pkg/buffer/writer.go
index 3d87a7b..0da0505 100644
--- a/pkg/buffer/writer.go
+++ b/pkg/buffer/writer.go
@@ -15,6 +15,9 @@
// specific language governing permissions and limitations
// under the License.
+// TODO:// move this into package encoding.
+
+// Package buffer implements a custom buffer based on a reusable binary array.
package buffer
import (
@@ -29,44 +32,53 @@ type Writer struct {
scratch [binary.MaxVarintLen64]byte
}
+// NewBufferWriter returns a new Writer.
func NewBufferWriter(buf *bytes.Buffer) *Writer {
return &Writer{
buf: buf,
}
}
+// Write binaries to the buffer.
func (w *Writer) Write(p []byte) {
_, _ = w.buf.Write(p)
}
+// WriteTo writes data to another Writer.
func (w *Writer) WriteTo(other *Writer) (n int64) {
n, _ = w.buf.WriteTo(other.buf)
return n
}
+// PutUint16 puts uint16 data into the buffer.
func (w *Writer) PutUint16(v uint16) {
binary.LittleEndian.PutUint16(w.scratch[:], v)
_, _ = w.buf.Write(w.scratch[:2])
}
+// PutUint32 puts uint32 data into the buffer.
func (w *Writer) PutUint32(v uint32) {
binary.LittleEndian.PutUint32(w.scratch[:], v)
_, _ = w.buf.Write(w.scratch[:4])
}
+// PutUint64 puts uint64 data into the buffer.
func (w *Writer) PutUint64(v uint64) {
binary.LittleEndian.PutUint64(w.scratch[:], v)
_, _ = w.buf.Write(w.scratch[:8])
}
+// Reset cleans up the buffer.
func (w *Writer) Reset() {
w.buf.Reset()
}
+// Len returns the length of the buffer.
func (w *Writer) Len() int {
return w.buf.Len()
}
+// Bytes outputs the data in the buffer.
func (w *Writer) Bytes() []byte {
return w.buf.Bytes()
}
diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go
index 90a4bc3..5e46c2f 100644
--- a/pkg/bus/bus.go
+++ b/pkg/bus/bus.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// Package bus implements a message bus which is a common data model and a messaging infrastructure
+// to allow different modules to communicate locally or remotely.
package bus
import (
@@ -27,11 +29,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-// Payload represents a simple data.
-type Payload interface{}
type (
+ payload interface{}
+
+ // MessageID the identity of a Message.
MessageID uint64
- Future interface {
+
+ // Future represents a future result of an asynchronous publishing.
+ Future interface {
Get() (Message, error)
GetAll() ([]Message, error)
}
@@ -39,18 +44,21 @@ type (
// Message is send on the bus to all subscribed listeners.
type Message struct {
- payload Payload
+ payload payload
id MessageID
}
+// ID outputs the MessageID of the Message.
func (m Message) ID() MessageID {
return m.id
}
+// Data returns the data wrapped in the Message.
func (m Message) Data() interface{} {
return m.payload
}
+// NewMessage returns a new Message with a MessageID and embed data.
func NewMessage(id MessageID, data interface{}) Message {
return Message{id: id, payload: data}
}
@@ -60,65 +68,73 @@ type MessageListener interface {
Rev(message Message) Message
}
+// Subscriber allow subscribing a Topic's messages.
type Subscriber interface {
Subscribe(topic Topic, listener MessageListener) error
}
+// Publisher allow sending Messages to a Topic.
type Publisher interface {
Publish(topic Topic, message ...Message) (Future, error)
}
-type Channel chan Event
+type channel chan event
-type ChType int
+type chType int
var (
- ChTypeUnidirectional ChType = 0
- ChTypeBidirectional ChType = 1
+ chTypeUnidirectional chType
+ chTypeBidirectional chType = 1
)
+// Topic is the object which messages are sent to or received from.
type Topic struct {
- ID string
- Type ChType
+ id string
+ typ chType
}
+// UniTopic returns an unary Topic.
func UniTopic(id string) Topic {
- return Topic{ID: id, Type: ChTypeUnidirectional}
+ return Topic{id: id, typ: chTypeUnidirectional}
}
+// BiTopic returns bidirectional Topic.
func BiTopic(id string) Topic {
- return Topic{ID: id, Type: ChTypeBidirectional}
+ return Topic{id: id, typ: chTypeBidirectional}
}
// The Bus allows publish-subscribe-style communication between components.
type Bus struct {
- topics map[Topic][]Channel
+ topics map[Topic][]channel
closer *run.Closer
mutex sync.RWMutex
}
+// NewBus returns a Bus.
func NewBus() *Bus {
b := new(Bus)
- b.topics = make(map[Topic][]Channel)
+ b.topics = make(map[Topic][]channel)
b.closer = run.NewCloser(0)
return b
}
var (
- ErrTopicEmpty = errors.New("the topic is empty")
+ // ErrTopicNotExist hints the topic published doesn't exist.
ErrTopicNotExist = errors.New("the topic does not exist")
- ErrListenerEmpty = errors.New("the message listener is empty")
- ErrEmptyFuture = errors.New("can't invoke Get() on an empty future")
+
+ errTopicEmpty = errors.New("the topic is empty")
+ errListenerEmpty = errors.New("the message listener is empty")
+ errEmptyFuture = errors.New("can't invoke Get() on an empty future")
)
type emptyFuture struct{}
func (e *emptyFuture) Get() (Message, error) {
- return Message{}, ErrEmptyFuture
+ return Message{}, errEmptyFuture
}
func (e *emptyFuture) GetAll() ([]Message, error) {
- return nil, ErrEmptyFuture
+ return nil, errEmptyFuture
}
type localFuture struct {
@@ -154,14 +170,15 @@ func (l *localFuture) GetAll() ([]Message, error) {
}
}
-type Event struct {
+type event struct {
f Future
m Message
}
+// Publish sends Messages to a Topic.
func (b *Bus) Publish(topic Topic, message ...Message) (Future, error) {
- if topic.ID == "" {
- return nil, ErrTopicEmpty
+ if topic.id == "" {
+ return nil, errTopicEmpty
}
b.mutex.RLock()
defer b.mutex.RUnlock()
@@ -170,15 +187,15 @@ func (b *Bus) Publish(topic Topic, message ...Message) (Future, error) {
return nil, ErrTopicNotExist
}
var f Future
- switch topic.Type {
- case ChTypeUnidirectional:
+ switch topic.typ {
+ case chTypeUnidirectional:
f = nil
- case ChTypeBidirectional:
+ case chTypeBidirectional:
f = &localFuture{retCount: len(message), retCh: make(chan Message)}
}
for _, each := range cc {
for _, m := range message {
- go func(ch Channel, message Message) {
+ go func(ch channel, message Message) {
if !b.closer.AddRunning() {
return
}
@@ -186,7 +203,7 @@ func (b *Bus) Publish(topic Topic, message ...Message) (Future, error) {
select {
case <-b.closer.CloseNotify():
return
- case ch <- Event{
+ case ch <- event{
m: message,
f: f,
}:
@@ -202,22 +219,22 @@ func (b *Bus) Publish(topic Topic, message ...Message) (Future, error) {
// Subscribe adds an MessageListener to be called when a message of a Topic is posted.
func (b *Bus) Subscribe(topic Topic, listener MessageListener) error {
- if topic.ID == "" {
- return ErrTopicEmpty
+ if topic.id == "" {
+ return errTopicEmpty
}
if listener == nil {
- return ErrListenerEmpty
+ return errListenerEmpty
}
b.mutex.Lock()
defer b.mutex.Unlock()
if _, exist := b.topics[topic]; !exist {
- b.topics[topic] = make([]Channel, 0)
+ b.topics[topic] = make([]channel, 0)
}
- ch := make(Channel)
+ ch := make(channel)
list := b.topics[topic]
list = append(list, ch)
b.topics[topic] = list
- go func(listener MessageListener, ch Channel) {
+ go func(listener MessageListener, ch channel) {
for {
c, ok := <-ch
if ok {
@@ -236,6 +253,7 @@ func (b *Bus) Subscribe(topic Topic, listener MessageListener) error {
return nil
}
+// Close a Bus until all Messages are sent to Subscribers.
func (b *Bus) Close() {
b.closer.CloseThenWait()
for _, chs := range b.topics {
diff --git a/pkg/bus/bus_test.go b/pkg/bus/bus_test.go
index 156e6f7..2007114 100644
--- a/pkg/bus/bus_test.go
+++ b/pkg/bus/bus_test.go
@@ -185,7 +185,7 @@ func TestBus_PubAndSub(t *testing.T) {
}
go func(want []MessageID) {
ret, errRet := f.GetAll()
- if errors.Is(errRet, ErrEmptyFuture) {
+ if errors.Is(errRet, errEmptyFuture) {
return
} else if errRet != nil {
t.Errorf("Publish()'s return message error = %v", err)
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 0ab6d9b..cc860d7 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package config implements a configuration system which could load configuration from flags and env vars.
package config
import (
@@ -37,15 +38,13 @@ type config struct {
name string
}
+// Load configurations from flags.
func Load(name string, fs *pflag.FlagSet) error {
c := new(config)
v := viper.New()
c.name = name
c.viper = v
- if err := c.initializeConfig(fs); err != nil {
- return err
- }
- return nil
+ return c.initializeConfig(fs)
}
func (c *config) initializeConfig(fs *pflag.FlagSet) error {
diff --git a/pkg/convert/hash.go b/pkg/convert/doc.go
similarity index 83%
copy from pkg/convert/hash.go
copy to pkg/convert/doc.go
index b08c2a5..00ddacc 100644
--- a/pkg/convert/hash.go
+++ b/pkg/convert/doc.go
@@ -15,14 +15,5 @@
// specific language governing permissions and limitations
// under the License.
+// Package convert implements conversions to and from data.
package convert
-
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
diff --git a/pkg/convert/hash.go b/pkg/convert/hash.go
index b08c2a5..94c05a9 100644
--- a/pkg/convert/hash.go
+++ b/pkg/convert/hash.go
@@ -19,10 +19,12 @@ package convert
import "github.com/cespare/xxhash"
+// Hash computes the 64-bit Hash digest of key.
func Hash(key []byte) uint64 {
return xxhash.Sum64(key)
}
+// HashStr computes the 64-bit Hash digest of key.
func HashStr(key string) uint64 {
return xxhash.Sum64String(key)
}
diff --git a/pkg/convert/number.go b/pkg/convert/number.go
index 9704d57..f94c252 100644
--- a/pkg/convert/number.go
+++ b/pkg/convert/number.go
@@ -21,12 +21,14 @@ import (
"encoding/binary"
)
+// Uint64ToBytes converts uint64 to bytes.
func Uint64ToBytes(u uint64) []byte {
bs := make([]byte, 8)
binary.BigEndian.PutUint64(bs, u)
return bs
}
+// Int64ToBytes converts int64 to bytes.
func Int64ToBytes(i int64) []byte {
abs := i
if i < 0 {
@@ -41,18 +43,14 @@ func Int64ToBytes(i int64) []byte {
return Uint64ToBytes(u)
}
-func Uint16ToBytes(u uint16) []byte {
- bs := make([]byte, 2)
- binary.BigEndian.PutUint16(bs, u)
- return bs
-}
-
+// Uint32ToBytes converts uint32 to bytes.
func Uint32ToBytes(u uint32) []byte {
bs := make([]byte, 4)
binary.BigEndian.PutUint32(bs, u)
return bs
}
+// BytesToInt64 converts bytes to int64.
func BytesToInt64(b []byte) int64 {
u := binary.BigEndian.Uint64(b)
if b[0] >= 128 {
@@ -67,46 +65,12 @@ func BytesToInt64(b []byte) int64 {
return abs
}
+// BytesToUint64 converts bytes to uint64.
func BytesToUint64(b []byte) uint64 {
return binary.BigEndian.Uint64(b)
}
+// BytesToUint32 converts bytes to uint32.
func BytesToUint32(b []byte) uint32 {
return binary.BigEndian.Uint32(b)
}
-
-func BytesToUint16(b []byte) uint16 {
- return binary.BigEndian.Uint16(b)
-}
-
-func IntToInt64(numbers ...int) []int64 {
- var arr []int64
- for i := 0; i < len(numbers); i++ {
- arr = append(arr, int64(numbers[i]))
- }
- return arr
-}
-
-func Int8ToInt64(numbers ...int8) []int64 {
- var arr []int64
- for i := 0; i < len(numbers); i++ {
- arr = append(arr, int64(numbers[i]))
- }
- return arr
-}
-
-func Int16ToInt64(numbers ...int16) []int64 {
- var arr []int64
- for i := 0; i < len(numbers); i++ {
- arr = append(arr, int64(numbers[i]))
- }
- return arr
-}
-
-func Int32ToInt64(numbers ...int32) []int64 {
- var arr []int64
- for i := 0; i < len(numbers); i++ {
- arr = append(arr, int64(numbers[i]))
- }
- return arr
-}
diff --git a/pkg/convert/number_test.go b/pkg/convert/number_test.go
index 0db986f..eceebe9 100644
--- a/pkg/convert/number_test.go
+++ b/pkg/convert/number_test.go
@@ -22,7 +22,8 @@ import (
"testing"
)
-func TestInt64ToBytes(t *testing.T) {
+// TODO: fix this case.
+func TestInt64ToBytes(_ *testing.T) {
fmt.Println(Int64ToBytes(-100))
fmt.Println(Int64ToBytes(-2))
fmt.Println(Int64ToBytes(-1))
diff --git a/pkg/encoding/encoding.go b/pkg/encoding/encoding.go
index 67f3b68..d3566ec 100644
--- a/pkg/encoding/encoding.go
+++ b/pkg/encoding/encoding.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package encoding implements encoding/decoding data points.
package encoding
import (
@@ -23,7 +24,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promauto"
)
-var ErrEncodeEmpty = errors.New("encode an empty value")
+var errEncodeEmpty = errors.New("encode an empty value")
var (
rawSize = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -43,6 +44,7 @@ var (
}, []string{"name", "type"})
)
+// SeriesEncoderPool allows putting and getting SeriesEncoder.
type SeriesEncoderPool interface {
Get(metadata []byte) SeriesEncoder
Put(encoder SeriesEncoder)
@@ -62,6 +64,7 @@ type SeriesEncoder interface {
StartTime() uint64
}
+// SeriesDecoderPool allows putting and getting SeriesDecoder.
type SeriesDecoderPool interface {
Get(metadata []byte) SeriesDecoder
Put(encoder SeriesDecoder)
diff --git a/pkg/encoding/int.go b/pkg/encoding/int.go
index 3b8522c..fc14d86 100644
--- a/pkg/encoding/int.go
+++ b/pkg/encoding/int.go
@@ -48,6 +48,7 @@ type intEncoderPoolDelegator struct {
size int
}
+// NewIntEncoderPool returns a SeriesEncoderPool which provides int-based xor encoders.
func NewIntEncoderPool(name string, size int, fn ParseInterval) SeriesEncoderPool {
return &intEncoderPoolDelegator{
name: name,
@@ -80,6 +81,7 @@ type intDecoderPoolDelegator struct {
size int
}
+// NewIntDecoderPool returns a SeriesDecoderPool which provides int-based xor decoders.
func NewIntDecoderPool(name string, size int, fn ParseInterval) SeriesDecoderPool {
return &intDecoderPoolDelegator{
name: name,
@@ -106,6 +108,7 @@ func (b *intDecoderPoolDelegator) Put(decoder SeriesDecoder) {
var _ SeriesEncoder = (*intEncoder)(nil)
+// ParseInterval parses the interval rule from the key in a kv pair.
type ParseInterval = func(key []byte) time.Duration
type intEncoder struct {
@@ -200,7 +203,7 @@ type intDecoder struct {
func (i *intDecoder) Decode(key, data []byte) error {
if len(data) < 10 {
- return ErrInvalidValue
+ return errInvalidValue
}
i.interval = i.fn(key)
i.startTime = binary.LittleEndian.Uint64(data[len(data)-10 : len(data)-2])
diff --git a/pkg/encoding/plain.go b/pkg/encoding/plain.go
index 50ecb10..504fc5a 100644
--- a/pkg/encoding/plain.go
+++ b/pkg/encoding/plain.go
@@ -47,6 +47,7 @@ type plainEncoderPoolDelegator struct {
size int
}
+// NewPlainEncoderPool returns a SeriesEncoderPool which provides encoders encoding arbitrary data.
func NewPlainEncoderPool(name string, size int) SeriesEncoderPool {
return &plainEncoderPoolDelegator{
name: name,
@@ -76,6 +77,7 @@ type plainDecoderPoolDelegator struct {
size int
}
+// NewPlainDecoderPool returns a NewPlainDecoderPool which provides decoders decoding arbitrary data.
func NewPlainDecoderPool(name string, size int) SeriesDecoderPool {
return &plainDecoderPoolDelegator{
name: name,
@@ -151,7 +153,7 @@ func (t *plainEncoder) Reset(_ []byte) {
func (t *plainEncoder) Encode() ([]byte, error) {
if t.tsBuff.Len() < 1 {
- return nil, ErrEncodeEmpty
+ return nil, errEncodeEmpty
}
val := t.valBuff.Bytes()
t.len = uint32(len(val))
@@ -185,7 +187,7 @@ const (
TSLen = 8 + 4
)
-var ErrInvalidValue = errors.New("invalid encoded value")
+var errInvalidValue = errors.New("invalid encoded value")
// plainDecoder decodes encoded time index.
type plainDecoder struct {
@@ -203,7 +205,7 @@ func (t *plainDecoder) Len() int {
func (t *plainDecoder) Decode(_, rawData []byte) (err error) {
if len(rawData) < 2 {
- return ErrInvalidValue
+ return errInvalidValue
}
var data []byte
size := binary.LittleEndian.Uint16(rawData[len(rawData)-2:])
@@ -212,14 +214,14 @@ func (t *plainDecoder) Decode(_, rawData []byte) (err error) {
}
l := uint32(len(data))
if l <= 8 {
- return ErrInvalidValue
+ return errInvalidValue
}
lenOffset := len(data) - 4
numOffset := lenOffset - 4
t.num = binary.LittleEndian.Uint32(data[numOffset:lenOffset])
t.len = binary.LittleEndian.Uint32(data[lenOffset:])
if l <= t.len+8 {
- return ErrInvalidValue
+ return errInvalidValue
}
t.val = data[:t.len]
t.ts = data[t.len:numOffset]
@@ -257,7 +259,7 @@ func (t *plainDecoder) Iterator() SeriesIterator {
func getVal(buf []byte, offset uint32) ([]byte, error) {
if uint32(len(buf)) <= offset+4 {
- return nil, ErrInvalidValue
+ return nil, errInvalidValue
}
dataLen := binary.LittleEndian.Uint32(buf[offset : offset+4])
return buf[offset+4 : offset+4+dataLen], nil
diff --git a/pkg/flow/dedup_priority_queue.go b/pkg/flow/dedup_priority_queue.go
index 86f9c91..527ab6f 100644
--- a/pkg/flow/dedup_priority_queue.go
+++ b/pkg/flow/dedup_priority_queue.go
@@ -40,6 +40,7 @@ type DedupPriorityQueue struct {
allowDuplicates bool
}
+// NewPriorityQueue returns a new DedupPriorityQueue.
func NewPriorityQueue(comparator utils.Comparator, allowDuplicates bool) *DedupPriorityQueue {
return &DedupPriorityQueue{
comparator: comparator,
@@ -101,11 +102,13 @@ func (pq *DedupPriorityQueue) Peek() Element {
return nil
}
+// ReplaceLowest replaces the lowest item with the newLowest.
func (pq *DedupPriorityQueue) ReplaceLowest(newLowest Element) {
pq.Items[0] = newLowest
heap.Fix(pq, 0)
}
+// Values returns all items.
func (pq *DedupPriorityQueue) Values() []Element {
values := make([]Element, pq.Len())
for pq.Len() > 0 {
diff --git a/pkg/flow/streaming/sink/slice.go b/pkg/flow/streaming/sink/slice.go
deleted file mode 100644
index b755aa5..0000000
--- a/pkg/flow/streaming/sink/slice.go
+++ /dev/null
@@ -1,82 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package sink
-
-import (
- "context"
- "sync"
-
- "github.com/apache/skywalking-banyandb/pkg/flow"
-)
-
-var _ flow.Sink = (*Slice)(nil)
-
-type Slice struct {
- in chan flow.StreamRecord
- slice []interface{}
- flow.ComponentState
- sync.RWMutex
-}
-
-func NewSlice() *Slice {
- return &Slice{
- slice: make([]interface{}, 0),
- in: make(chan flow.StreamRecord),
- }
-}
-
-func (s *Slice) Value() []interface{} {
- s.RLock()
- defer s.RUnlock()
- return s.slice
-}
-
-func (s *Slice) In() chan<- flow.StreamRecord {
- return s.in
-}
-
-func (s *Slice) Setup(ctx context.Context) error {
- go s.run(ctx)
-
- return nil
-}
-
-func (s *Slice) run(ctx context.Context) {
- s.Add(1)
- defer func() {
- s.Done()
- }()
- for {
- select {
- case item, ok := <-s.in:
- if !ok {
- return
- }
- s.Lock()
- s.slice = append(s.slice, item)
- s.Unlock()
- case <-ctx.Done():
- return
- }
- }
-}
-
-func (s *Slice) Teardown(ctx context.Context) error {
- s.Wait()
- return nil
-}
diff --git a/pkg/flow/streaming/flow_sliding_window.go b/pkg/flow/streaming/sliding_window.go
similarity index 83%
rename from pkg/flow/streaming/flow_sliding_window.go
rename to pkg/flow/streaming/sliding_window.go
index 853ce55..860f193 100644
--- a/pkg/flow/streaming/flow_sliding_window.go
+++ b/pkg/flow/streaming/sliding_window.go
@@ -30,24 +30,24 @@ import (
"github.com/apache/skywalking-banyandb/pkg/flow"
)
-type TriggerResult bool
+type triggerResult bool
const (
- FIRE TriggerResult = true
- CONTINUE = false
+ fire triggerResult = true
+ cont = false
)
var (
- _ flow.Operator = (*TumblingTimeWindows)(nil)
- _ flow.WindowAssigner = (*TumblingTimeWindows)(nil)
+ _ flow.Operator = (*tumblingTimeWindows)(nil)
+ _ flow.WindowAssigner = (*tumblingTimeWindows)(nil)
_ flow.Window = (*timeWindow)(nil)
- DefaultCacheSize = 2
+ defaultCacheSize = 2
)
func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
switch v := w.(type) {
- case *TumblingTimeWindows:
+ case *tumblingTimeWindows:
v.errorHandler = f.drainErr
f.ops = append(f.ops, v)
default:
@@ -62,7 +62,7 @@ func (f *streamingFlow) Window(w flow.WindowAssigner) flow.WindowedFlow {
func (s *windowedFlow) AllowedMaxWindows(windowCnt int) flow.WindowedFlow {
switch v := s.wa.(type) {
- case *TumblingTimeWindows:
+ case *tumblingTimeWindows:
v.windowCount = windowCnt
default:
s.f.drainErr(errors.New("windowCnt is not supported"))
@@ -70,7 +70,7 @@ func (s *windowedFlow) AllowedMaxWindows(windowCnt int) flow.WindowedFlow {
return s
}
-type TumblingTimeWindows struct {
+type tumblingTimeWindows struct {
errorHandler func(error)
snapshots *lru.Cache
timerHeap *flow.DedupPriorityQueue
@@ -84,18 +84,18 @@ type TumblingTimeWindows struct {
timerMu sync.Mutex
}
-func (s *TumblingTimeWindows) In() chan<- flow.StreamRecord {
+func (s *tumblingTimeWindows) In() chan<- flow.StreamRecord {
return s.in
}
-func (s *TumblingTimeWindows) Out() <-chan flow.StreamRecord {
+func (s *tumblingTimeWindows) Out() <-chan flow.StreamRecord {
return s.out
}
-func (s *TumblingTimeWindows) Setup(ctx context.Context) (err error) {
+func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
if s.snapshots == nil {
if s.windowCount <= 0 {
- s.windowCount = DefaultCacheSize
+ s.windowCount = defaultCacheSize
}
s.snapshots, err = lru.NewWithEvict(s.windowCount, func(key interface{}, value interface{}) {
s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
@@ -111,19 +111,19 @@ func (s *TumblingTimeWindows) Setup(ctx context.Context) (err error) {
return
}
-func (s *TumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) {
+func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.AggregationOp) {
if snapshot.Dirty() {
s.out <- flow.NewStreamRecord(snapshot.Snapshot(), w.start)
}
}
-func (s *TumblingTimeWindows) flushWindow(w timeWindow) {
+func (s *tumblingTimeWindows) flushWindow(w timeWindow) {
if snapshot, ok := s.snapshots.Get(w); ok {
s.flushSnapshot(w, snapshot.(flow.AggregationOp))
}
}
-func (s *TumblingTimeWindows) flushDueWindows() {
+func (s *tumblingTimeWindows) flushDueWindows() {
s.timerMu.Lock()
defer s.timerMu.Unlock()
for {
@@ -138,13 +138,13 @@ func (s *TumblingTimeWindows) flushDueWindows() {
}
}
-func (s *TumblingTimeWindows) flushDirtyWindows() {
+func (s *tumblingTimeWindows) flushDirtyWindows() {
for _, key := range s.snapshots.Keys() {
s.flushWindow(key.(timeWindow))
}
}
-func (s *TumblingTimeWindows) receive() {
+func (s *tumblingTimeWindows) receive() {
defer s.Done()
for elem := range s.in {
@@ -173,7 +173,7 @@ func (s *TumblingTimeWindows) receive() {
}
result := ctx.OnElement(elem)
- if result == FIRE {
+ if result == fire {
s.flushWindow(tw)
}
}
@@ -211,22 +211,23 @@ func (s *TumblingTimeWindows) receive() {
// 1) the max timestamp is before the current watermark
// 2) the LRU cache is full
// 3) the LRU cache does not contain the window entry.
-func (s *TumblingTimeWindows) isWindowLate(w flow.Window) bool {
+func (s *tumblingTimeWindows) isWindowLate(w flow.Window) bool {
return w.MaxTimestamp() <= s.currentWatermark && s.snapshots.Len() >= s.windowCount && !s.snapshots.Contains(w)
}
-func (s *TumblingTimeWindows) Teardown(ctx context.Context) error {
+func (s *tumblingTimeWindows) Teardown(_ context.Context) error {
s.Wait()
return nil
}
-func (s *TumblingTimeWindows) Exec(downstream flow.Inlet) {
+func (s *tumblingTimeWindows) Exec(downstream flow.Inlet) {
s.Add(1)
go flow.Transmit(&s.ComponentState, downstream, s)
}
-func NewTumblingTimeWindows(size time.Duration) *TumblingTimeWindows {
- return &TumblingTimeWindows{
+// NewTumblingTimeWindows return tumbling-time windows.
+func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
+ return &tumblingTimeWindows{
windowSize: size.Milliseconds(),
timerHeap: flow.NewPriorityQueue(func(a, b interface{}) int {
return int(a.(*internalTimer).triggerTimeMillis - b.(*internalTimer).triggerTimeMillis)
@@ -247,7 +248,7 @@ func (t timeWindow) MaxTimestamp() int64 {
}
// AssignWindows assigns windows according to the given timestamp.
-func (s *TumblingTimeWindows) AssignWindows(timestamp int64) ([]flow.Window, error) {
+func (s *tumblingTimeWindows) AssignWindows(timestamp int64) ([]flow.Window, error) {
if timestamp > math.MinInt64 {
start := getWindowStart(timestamp, s.windowSize)
return []flow.Window{
@@ -267,17 +268,17 @@ func getWindowStart(timestamp, windowSize int64) int64 {
}
// eventTimeTriggerOnElement processes element(s) with EventTimeTrigger.
-func eventTimeTriggerOnElement(window timeWindow, ctx *triggerContext) TriggerResult {
+func eventTimeTriggerOnElement(window timeWindow, ctx *triggerContext) triggerResult {
if window.MaxTimestamp() <= ctx.GetCurrentWatermark() {
// if watermark is already past the window fire immediately
- return FIRE
+ return fire
}
ctx.RegisterEventTimeTimer(window.MaxTimestamp())
- return CONTINUE
+ return cont
}
type triggerContext struct {
- delegation *TumblingTimeWindows
+ delegation *tumblingTimeWindows
window timeWindow
}
@@ -294,7 +295,7 @@ func (ctx *triggerContext) RegisterEventTimeTimer(triggerTime int64) {
})
}
-func (ctx *triggerContext) OnElement(record flow.StreamRecord) TriggerResult {
+func (ctx *triggerContext) OnElement(_ flow.StreamRecord) triggerResult {
return eventTimeTriggerOnElement(ctx.window, ctx)
}
diff --git a/pkg/flow/streaming/flow_sliding_window_test.go b/pkg/flow/streaming/sliding_window_test.go
similarity index 94%
rename from pkg/flow/streaming/flow_sliding_window_test.go
rename to pkg/flow/streaming/sliding_window_test.go
index 8672527..83f9887 100644
--- a/pkg/flow/streaming/flow_sliding_window_test.go
+++ b/pkg/flow/streaming/sliding_window_test.go
@@ -25,7 +25,6 @@ import (
. "github.com/onsi/gomega"
"github.com/apache/skywalking-banyandb/pkg/flow"
- "github.com/apache/skywalking-banyandb/pkg/flow/streaming/sink"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
)
@@ -57,9 +56,9 @@ func (i *intSumAggregator) Dirty() bool {
var _ = Describe("Sliding Window", func() {
var (
baseTS time.Time
- snk *sink.Slice
+ snk *slice
input []flow.StreamRecord
- slidingWindows *TumblingTimeWindows
+ slidingWindows *tumblingTimeWindows
aggrFactory = func() flow.AggregationOp {
return &intSumAggregator{}
@@ -71,9 +70,9 @@ var _ = Describe("Sliding Window", func() {
})
JustBeforeEach(func() {
- snk = sink.NewSlice()
+ snk = newSlice()
- slidingWindows = NewTumblingTimeWindows(time.Second * 15)
+ slidingWindows = NewTumblingTimeWindows(time.Second * 15).(*tumblingTimeWindows)
slidingWindows.aggregationFactory = aggrFactory
slidingWindows.windowCount = 2
diff --git a/pkg/flow/streaming/sources/channel.go b/pkg/flow/streaming/sources/channel.go
index 5ab7c7b..ddc2843 100644
--- a/pkg/flow/streaming/sources/channel.go
+++ b/pkg/flow/streaming/sources/channel.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package sources implements data sources to sink data into the flow framework.
package sources
import (
@@ -29,7 +30,7 @@ import (
var _ flow.Source = (*sourceChan)(nil)
type sourceChan struct {
- in interface{}
+ in chan any
out chan flow.StreamRecord
flow.ComponentState
}
@@ -39,18 +40,12 @@ func (s *sourceChan) Out() <-chan flow.StreamRecord {
}
func (s *sourceChan) Setup(ctx context.Context) error {
- chanVal := reflect.ValueOf(s.in)
-
- if !chanVal.IsValid() {
- return errors.New("invalid channel")
- }
-
s.Add(1)
- go s.run(ctx, chanVal)
+ go s.run(ctx)
return nil
}
-func (s *sourceChan) run(ctx context.Context, chanVal reflect.Value) {
+func (s *sourceChan) run(ctx context.Context) {
ctx, cancel := context.WithCancel(ctx)
defer func() {
cancel()
@@ -58,20 +53,16 @@ func (s *sourceChan) run(ctx context.Context, chanVal reflect.Value) {
s.Done()
}()
- for {
- val, open := chanVal.Recv()
- if !open {
- return
- }
+ for val := range s.in {
select {
- case s.out <- flow.TryExactTimestamp(val.Interface()):
+ case s.out <- flow.TryExactTimestamp(val):
case <-ctx.Done():
return
}
}
}
-func (s *sourceChan) Teardown(ctx context.Context) error {
+func (s *sourceChan) Teardown(_ context.Context) error {
s.Wait()
return nil
}
@@ -81,7 +72,8 @@ func (s *sourceChan) Exec(downstream flow.Inlet) {
go flow.Transmit(&s.ComponentState, downstream, s)
}
-func NewChannel(in interface{}) (flow.Source, error) {
+// NewChannel returns a source from in which should be a cannel.
+func NewChannel(in chan any) (flow.Source, error) {
if reflect.TypeOf(in).Kind() != reflect.Chan {
return nil, errors.New("in must be a Channel")
}
diff --git a/pkg/flow/streaming/sources/channel_test.go b/pkg/flow/streaming/sources/channel_test.go
index 984d513..40cc5fe 100644
--- a/pkg/flow/streaming/sources/channel_test.go
+++ b/pkg/flow/streaming/sources/channel_test.go
@@ -39,7 +39,7 @@ func TestSource_channel(t *testing.T) {
inlet := flow.NewMockInlet(ctrl)
- srcCh := make(chan string)
+ srcCh := make(chan any)
assert := require.New(t)
src, err := sources.NewChannel(srcCh)
diff --git a/pkg/flow/streaming/flow.go b/pkg/flow/streaming/streaming.go
similarity index 94%
rename from pkg/flow/streaming/flow.go
rename to pkg/flow/streaming/streaming.go
index 8bb6251..63b02ed 100644
--- a/pkg/flow/streaming/flow.go
+++ b/pkg/flow/streaming/streaming.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package streaming implement the flow framework to provide the sliding window, top-n aggregation, and etc.
package streaming
import (
@@ -36,6 +37,7 @@ type streamingFlow struct {
ops []flow.Operator
}
+// New returns a new streaming flow.
func New(source flow.Source) flow.Flow {
return &streamingFlow{
source: source,
diff --git a/pkg/flow/streaming/flow_test.go b/pkg/flow/streaming/streaming_test.go
similarity index 89%
rename from pkg/flow/streaming/flow_test.go
rename to pkg/flow/streaming/streaming_test.go
index 5ca82dd..a89e10e 100644
--- a/pkg/flow/streaming/flow_test.go
+++ b/pkg/flow/streaming/streaming_test.go
@@ -19,13 +19,13 @@ package streaming
import (
"context"
+ "sync"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/apache/skywalking-banyandb/pkg/flow"
- "github.com/apache/skywalking-banyandb/pkg/flow/streaming/sink"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
flowTest "github.com/apache/skywalking-banyandb/pkg/test/flow"
)
@@ -43,7 +43,7 @@ func numberRange(begin, count int) []int {
var _ = Describe("Streaming", func() {
var (
f flow.Flow
- snk *sink.Slice
+ snk *slice
errCh <-chan error
)
@@ -60,7 +60,7 @@ var _ = Describe("Streaming", func() {
)
JustBeforeEach(func() {
- snk = sink.NewSlice()
+ snk = newSlice()
f = New(input).
Filter(filter).
To(snk)
@@ -97,7 +97,7 @@ var _ = Describe("Streaming", func() {
)
JustBeforeEach(func() {
- snk = sink.NewSlice()
+ snk = newSlice()
f = New(input).
Map(mapper).
To(snk)
@@ -141,7 +141,7 @@ var _ = Describe("Streaming", func() {
var input []flow.StreamRecord
JustBeforeEach(func() {
- snk = sink.NewSlice()
+ snk = newSlice()
f = New(flowTest.NewSlice(input)).
Map(flow.UnaryFunc[any](func(ctx context.Context, item interface{}) interface{} {
@@ -195,7 +195,7 @@ var _ = Describe("Streaming", func() {
var input []flow.StreamRecord
JustBeforeEach(func() {
- snk = sink.NewSlice()
+ snk = newSlice()
f = New(flowTest.NewSlice(input)).
Map(flow.UnaryFunc[any](func(ctx context.Context, item interface{}) interface{} {
@@ -239,3 +239,60 @@ var _ = Describe("Streaming", func() {
})
})
})
+
+var _ flow.Sink = (*slice)(nil)
+
+type slice struct {
+ in chan flow.StreamRecord
+ slice []interface{}
+ flow.ComponentState
+ sync.RWMutex
+}
+
+func newSlice() *slice {
+ return &slice{
+ slice: make([]interface{}, 0),
+ in: make(chan flow.StreamRecord),
+ }
+}
+
+func (s *slice) Value() []interface{} {
+ s.RLock()
+ defer s.RUnlock()
+ return s.slice
+}
+
+func (s *slice) In() chan<- flow.StreamRecord {
+ return s.in
+}
+
+func (s *slice) Setup(ctx context.Context) error {
+ go s.run(ctx)
+
+ return nil
+}
+
+func (s *slice) run(ctx context.Context) {
+ s.Add(1)
+ defer func() {
+ s.Done()
+ }()
+ for {
+ select {
+ case item, ok := <-s.in:
+ if !ok {
+ return
+ }
+ s.Lock()
+ s.slice = append(s.slice, item)
+ s.Unlock()
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (s *slice) Teardown(_ context.Context) error {
+ s.Wait()
+ return nil
+}
diff --git a/pkg/flow/streaming/flow_topn.go b/pkg/flow/streaming/topn.go
similarity index 91%
rename from pkg/flow/streaming/flow_topn.go
rename to pkg/flow/streaming/topn.go
index 7be2fa4..de63c7d 100644
--- a/pkg/flow/streaming/flow_topn.go
+++ b/pkg/flow/streaming/topn.go
@@ -20,14 +20,15 @@ package streaming
import (
"github.com/emirpasic/gods/maps/treemap"
"github.com/emirpasic/gods/utils"
- "github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
"github.com/apache/skywalking-banyandb/pkg/flow"
)
+// TopNSort defines the order of sorting.
type TopNSort uint8
+// The available order of sorting.
const (
DESC TopNSort = iota
ASC
@@ -39,7 +40,7 @@ type windowedFlow struct {
}
func (s *windowedFlow) TopN(topNum int, opts ...any) flow.Flow {
- s.wa.(*TumblingTimeWindows).aggregationFactory = func() flow.AggregationOp {
+ s.wa.(*tumblingTimeWindows).aggregationFactory = func() flow.AggregationOp {
topNAggrFunc := &topNAggregator{
cacheSize: topNum,
sort: DESC,
@@ -77,14 +78,17 @@ type topNAggregator struct {
dirty bool
}
+// TopNOption is the option to set up a top-n aggregator.
type TopNOption func(aggregator *topNAggregator)
+// WithSortKeyExtractor sets a closure to extract the sorting key.
func WithSortKeyExtractor(sortKeyExtractor func(flow.StreamRecord) int64) TopNOption {
return func(aggregator *topNAggregator) {
aggregator.sortKeyExtractor = sortKeyExtractor
}
}
+// OrderBy sets the sorting order.
func OrderBy(sort TopNSort) TopNOption {
return func(aggregator *topNAggregator) {
aggregator.sort = sort
@@ -144,15 +148,6 @@ func (t *topNAggregator) checkSortKeyInBufferRange(sortKey int64) bool {
return t.currentTopNum < t.cacheSize
}
-type Tuple2 struct {
- V1 interface{} `json:"v1"`
- V2 interface{} `json:"v2"`
-}
-
-func (t *Tuple2) Equal(other *Tuple2) bool {
- return cmp.Equal(t.V1, other.V1) && cmp.Equal(t.V2, other.V2)
-}
-
func (t *topNAggregator) Snapshot() interface{} {
t.dirty = false
iter := t.treeMap.Iterator()
@@ -169,3 +164,9 @@ func (t *topNAggregator) Snapshot() interface{} {
func (t *topNAggregator) Dirty() bool {
return t.dirty
}
+
+// Tuple2 is a tuple with 2 fields. Each field may be a separate type.
+type Tuple2 struct {
+ V1 interface{} `json:"v1"`
+ V2 interface{} `json:"v2"`
+}
diff --git a/pkg/flow/streaming/flow_topn_test.go b/pkg/flow/streaming/topn_test.go
similarity index 100%
rename from pkg/flow/streaming/flow_topn_test.go
rename to pkg/flow/streaming/topn_test.go
diff --git a/pkg/flow/streaming/flow_unary.go b/pkg/flow/streaming/unary.go
similarity index 97%
rename from pkg/flow/streaming/flow_unary.go
rename to pkg/flow/streaming/unary.go
index 22a8ede..6ec9419 100644
--- a/pkg/flow/streaming/flow_unary.go
+++ b/pkg/flow/streaming/unary.go
@@ -65,7 +65,7 @@ func (u *unaryOperator) Exec(downstream flow.Inlet) {
go flow.Transmit(&u.ComponentState, downstream, u)
}
-func (u *unaryOperator) Teardown(ctx context.Context) error {
+func (u *unaryOperator) Teardown(_ context.Context) error {
u.Wait()
return nil
}
diff --git a/pkg/flow/types.go b/pkg/flow/types.go
index 5b61910..778f9a7 100644
--- a/pkg/flow/types.go
+++ b/pkg/flow/types.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package flow implements a streaming calculation framework.
package flow
import (
@@ -25,6 +26,7 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
)
+// Data indicates a aggregated data.
type Data []any
// Flow is an abstraction of data flow for
@@ -82,6 +84,7 @@ type AggregationOp interface {
Dirty() bool
}
+// AggregationOpFactory is a factory to create AggregationOp.
type AggregationOpFactory func() AggregationOp
// StreamRecord is a container wraps user data and timestamp.
@@ -91,6 +94,7 @@ type StreamRecord struct {
ts int64
}
+// NewStreamRecord returns a StreamRecord with data and timestamp.
func NewStreamRecord(data interface{}, ts int64) StreamRecord {
return StreamRecord{
data: data,
@@ -98,6 +102,7 @@ func NewStreamRecord(data interface{}, ts int64) StreamRecord {
}
}
+// NewStreamRecordWithTimestampPb returns a StreamRecord whose timestamp is parsed from protobuf's timestamp.
func NewStreamRecordWithTimestampPb(data interface{}, timestamp *timestamppb.Timestamp) StreamRecord {
return StreamRecord{
data: data,
@@ -105,6 +110,7 @@ func NewStreamRecordWithTimestampPb(data interface{}, timestamp *timestamppb.Tim
}
}
+// NewStreamRecordWithoutTS returns a StreamRecord with data only.
func NewStreamRecordWithoutTS(data interface{}) StreamRecord {
return StreamRecord{
data: data,
@@ -112,6 +118,7 @@ func NewStreamRecordWithoutTS(data interface{}) StreamRecord {
}
}
+// WithNewData sets data to StreamRecord.
func (sr StreamRecord) WithNewData(data interface{}) StreamRecord {
return StreamRecord{
ts: sr.ts,
@@ -119,10 +126,12 @@ func (sr StreamRecord) WithNewData(data interface{}) StreamRecord {
}
}
+// TimestampMillis returns the timestamp in millisecond.
func (sr StreamRecord) TimestampMillis() int64 {
return sr.ts
}
+// Data returns the embedded data.
func (sr StreamRecord) Data() interface{} {
return sr.data
}
@@ -139,6 +148,7 @@ type Outlet interface {
Out() <-chan StreamRecord
}
+// Component is a lifecycle controller.
type Component interface {
// Setup is the lifecycle hook for resource preparation, e.g. start background job for listening input channel.
// It must be called before the flow starts to process elements.
@@ -148,6 +158,7 @@ type Component interface {
Teardown(context.Context) error
}
+// ComponentState watches whether the component is shutdown.
type ComponentState struct {
sync.WaitGroup
}
diff --git a/pkg/flow/utils.go b/pkg/flow/utils.go
index 7f4d805..b477a9f 100644
--- a/pkg/flow/utils.go
+++ b/pkg/flow/utils.go
@@ -27,6 +27,7 @@ func Transmit(state *ComponentState, downstream Inlet, current Outlet) {
close(downstream.In())
}
+// TryExactTimestamp tries to get the timestamp from the item.
func TryExactTimestamp(item any) StreamRecord {
if r, ok := item.(StreamRecord); ok {
return r
diff --git a/pkg/grpchelper/client.go b/pkg/grpchelper/client.go
index f482ebd..4e57877 100644
--- a/pkg/grpchelper/client.go
+++ b/pkg/grpchelper/client.go
@@ -14,6 +14,8 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
+// Package grpchelper implements helpers to access gRPC services.
package grpchelper
import (
@@ -29,6 +31,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
+// Conn returns a gRPC client connection once connecting the server.
func Conn(addr string, connTimeout time.Duration, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
defaultOpts := []grpc.DialOption{
grpc.WithBlock(),
@@ -55,6 +58,8 @@ func Conn(addr string, connTimeout time.Duration, opts ...grpc.DialOption) (*grp
return conn, nil
}
+// Request execute a input closure to send traffics.
+// It provides common features like timeout, error handling, and etc.
func Request(ctx context.Context, rpcTimeout time.Duration, fn func(rpcCtx context.Context) error) error {
rpcStart := time.Now()
rpcCtx, rpcCancel := context.WithTimeout(ctx, rpcTimeout)
diff --git a/pkg/index/index.go b/pkg/index/index.go
index 03d28ad..36849e6 100644
--- a/pkg/index/index.go
+++ b/pkg/index/index.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package index implements the index system for searching data.
package index
import (
@@ -32,18 +33,21 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index/posting"
)
-var ErrMalformed = errors.New("the data is malformed")
+var errMalformed = errors.New("the data is malformed")
+// FieldKey is the key of field in a document.
type FieldKey struct {
SeriesID common.SeriesID
IndexRuleID uint32
Analyzer databasev1.IndexRule_Analyzer
}
+// MarshalIndexRule encodes the index rule id to string representation.
func (f FieldKey) MarshalIndexRule() string {
return string(convert.Uint32ToBytes(f.IndexRuleID))
}
+// Marshal encodes f to bytes.
func (f FieldKey) Marshal() []byte {
return bytes.Join([][]byte{
f.SeriesID.Marshal(),
@@ -51,32 +55,38 @@ func (f FieldKey) Marshal() []byte {
}, nil)
}
+// MarshalToStr encodes f to string.
func (f FieldKey) MarshalToStr() string {
return string(f.Marshal())
}
+// Unmarshal decodes bytes to f.
func (f *FieldKey) Unmarshal(raw []byte) error {
f.SeriesID = common.SeriesID(convert.BytesToUint64(raw[0:8]))
f.IndexRuleID = convert.BytesToUint32(raw[8:])
return nil
}
+// Equal reports whether f and other have the same series id and index rule id.
func (f FieldKey) Equal(other FieldKey) bool {
return f.SeriesID == other.SeriesID && f.IndexRuleID == other.IndexRuleID
}
+// Field is a indexed item in a document.
type Field struct {
Term []byte
Key FieldKey
}
+// Marshal encodes f to bytes.
func (f Field) Marshal() ([]byte, error) {
return bytes.Join([][]byte{f.Key.Marshal(), f.Term}, nil), nil
}
+// Unmarshal decodes bytes to f.
func (f *Field) Unmarshal(raw []byte) error {
if len(raw) < 13 {
- return errors.WithMessagef(ErrMalformed, "malformed field: expected more than 12, got %d", len(raw))
+ return errors.WithMessagef(errMalformed, "malformed field: expected more than 12, got %d", len(raw))
}
fk := &f.Key
err := fk.Unmarshal(raw[:12])
@@ -89,6 +99,7 @@ func (f *Field) Unmarshal(raw []byte) error {
return nil
}
+// RangeOpts contains options to performance a continuous scan.
type RangeOpts struct {
Upper []byte
Lower []byte
@@ -96,6 +107,7 @@ type RangeOpts struct {
IncludesLower bool
}
+// Between reports whether value is in the range.
func (r RangeOpts) Between(value []byte) int {
if r.Upper != nil {
var in bool
@@ -122,41 +134,47 @@ func (r RangeOpts) Between(value []byte) int {
return 0
}
+// FieldIterator allows iterating over a field's posting values.
type FieldIterator interface {
Next() bool
Val() *PostingValue
Close() error
}
-var EmptyFieldIterator = &emptyIterator{}
+// DummyFieldIterator never iterates.
+var DummyFieldIterator = &dummyIterator{}
-type emptyIterator struct{}
+type dummyIterator struct{}
-func (i *emptyIterator) Next() bool {
+func (i *dummyIterator) Next() bool {
return false
}
-func (i *emptyIterator) Val() *PostingValue {
+func (i *dummyIterator) Val() *PostingValue {
return nil
}
-func (i *emptyIterator) Close() error {
+func (i *dummyIterator) Close() error {
return nil
}
+// PostingValue is the collection of a field's values.
type PostingValue struct {
Value posting.List
Term []byte
}
+// Writer allows writing fields and itemID in a document to a index.
type Writer interface {
Write(fields []Field, itemID common.ItemID) error
}
+// FieldIterable allows building a FieldIterator.
type FieldIterable interface {
Iterator(fieldKey FieldKey, termRange RangeOpts, order modelv1.Sort) (iter FieldIterator, err error)
}
+// Searcher allows searching a field either by its key or by its key and term.
type Searcher interface {
FieldIterable
Match(fieldKey FieldKey, match []string) (list posting.List, err error)
@@ -165,6 +183,7 @@ type Searcher interface {
Range(fieldKey FieldKey, opts RangeOpts) (list posting.List, err error)
}
+// Store is an abstract of a index repository.
type Store interface {
observability.Observable
io.Closer
@@ -172,8 +191,10 @@ type Store interface {
Searcher
}
+// GetSearcher returns a searcher associated with input index rule type.
type GetSearcher func(location databasev1.IndexRule_Type) (Searcher, error)
+// Filter is a node in the filter tree.
type Filter interface {
fmt.Stringer
Execute(getSearcher GetSearcher, seriesID common.SeriesID) (posting.List, error)
diff --git a/pkg/index/inverted/inverted.go b/pkg/index/inverted/inverted.go
index a77ab73..d1efa79 100644
--- a/pkg/index/inverted/inverted.go
+++ b/pkg/index/inverted/inverted.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package inverted implements a inverted index repository.
package inverted
import (
@@ -61,6 +62,7 @@ func init() {
var _ index.Store = (*store)(nil)
+// StoreOpts wraps options to create a inverted index repository.
type StoreOpts struct {
Logger *logger.Logger
Path string
@@ -82,6 +84,7 @@ type store struct {
l *logger.Logger
}
+// NewStore create a new inverted index repository.
func NewStore(opts StoreOpts) (index.Store, error) {
config := bluge.DefaultConfig(opts.Path)
config.DefaultSearchAnalyzer = analyzers[databasev1.IndexRule_ANALYZER_KEYWORD]
@@ -131,7 +134,7 @@ func (s *store) Iterator(fieldKey index.FieldKey, termRange index.RangeOpts, ord
if termRange.Lower != nil &&
termRange.Upper != nil &&
bytes.Compare(termRange.Lower, termRange.Upper) > 0 {
- return index.EmptyFieldIterator, nil
+ return index.DummyFieldIterator, nil
}
reader, err := s.writer.Reader()
if err != nil {
@@ -189,7 +192,7 @@ func (s *store) MatchTerms(field index.Field) (list posting.List, err error) {
func (s *store) Match(fieldKey index.FieldKey, matches []string) (posting.List, error) {
if len(matches) == 0 {
- return roaring.EmptyPostingList, nil
+ return roaring.DummyPostingList, nil
}
reader, err := s.writer.Reader()
if err != nil {
@@ -221,7 +224,7 @@ func (s *store) Match(fieldKey index.FieldKey, matches []string) (posting.List,
func (s *store) Range(fieldKey index.FieldKey, opts index.RangeOpts) (list posting.List, err error) {
iter, err := s.Iterator(fieldKey, opts, modelv1.Sort_SORT_ASC)
if err != nil {
- return roaring.EmptyPostingList, err
+ return roaring.DummyPostingList, err
}
list = roaring.NewPostingList()
for iter.Next() {
@@ -324,8 +327,10 @@ func (bmi *blugeMatchIterator) Next() bool {
if bmi.err != nil || bmi.closed {
return false
}
+ //revive:disable:empty-block
for bmi.nextTerm() {
}
+ //revive:enable:empty-block
if bmi.err != nil || bmi.closed {
return false
}
diff --git a/pkg/index/iterator.go b/pkg/index/lsm/iterator.go
similarity index 59%
rename from pkg/index/iterator.go
rename to pkg/index/lsm/iterator.go
index 0f51300..694e7ab 100644
--- a/pkg/index/iterator.go
+++ b/pkg/index/lsm/iterator.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package index
+package lsm
import (
"bytes"
@@ -26,29 +26,30 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/kv"
"github.com/apache/skywalking-banyandb/pkg/convert"
+ "github.com/apache/skywalking-banyandb/pkg/index"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
-type CompositePostingValueFn = func(term, value []byte, delegated kv.Iterator) (*PostingValue, error)
+type compositePostingValueFn = func(term, value []byte, delegated kv.Iterator) (*index.PostingValue, error)
var (
- _ FieldIterator = (*FieldIteratorTemplate)(nil)
- DefaultUpper = convert.Uint64ToBytes(math.MaxUint64)
- DefaultLower = convert.Uint64ToBytes(0)
+ _ index.FieldIterator = (*fieldIteratorTemplate)(nil)
+ defaultUpper = convert.Uint64ToBytes(math.MaxUint64)
+ defaultLower = convert.Uint64ToBytes(0)
)
-type FieldIteratorTemplate struct {
+type fieldIteratorTemplate struct {
err error
delegated *delegateIterator
- cur *PostingValue
- fn CompositePostingValueFn
+ cur *index.PostingValue
+ fn compositePostingValueFn
seekKey []byte
- termRange RangeOpts
+ termRange index.RangeOpts
init bool
reverse bool
}
-func (f *FieldIteratorTemplate) Next() bool {
+func (f *fieldIteratorTemplate) Next() bool {
if !f.init {
f.init = true
f.delegated.Seek(f.seekKey)
@@ -78,22 +79,22 @@ func (f *FieldIteratorTemplate) Next() bool {
return true
}
-func (f *FieldIteratorTemplate) Val() *PostingValue {
+func (f *fieldIteratorTemplate) Val() *index.PostingValue {
return f.cur
}
-func (f *FieldIteratorTemplate) Close() error {
+func (f *fieldIteratorTemplate) Close() error {
return multierr.Append(f.err, f.delegated.Close())
}
-func NewFieldIteratorTemplate(l *logger.Logger, fieldKey FieldKey, termRange RangeOpts, order modelv1.Sort, iterable kv.Iterable,
- fn CompositePostingValueFn,
-) (*FieldIteratorTemplate, error) {
+func newFieldIteratorTemplate(l *logger.Logger, fieldKey index.FieldKey, termRange index.RangeOpts, order modelv1.Sort, iterable kv.Iterable,
+ fn compositePostingValueFn,
+) (*fieldIteratorTemplate, error) {
if termRange.Upper == nil {
- termRange.Upper = DefaultUpper
+ termRange.Upper = defaultUpper
}
if termRange.Lower == nil {
- termRange.Lower = DefaultLower
+ termRange.Lower = defaultLower
}
var reverse bool
var term []byte
@@ -112,7 +113,7 @@ func NewFieldIteratorTemplate(l *logger.Logger, fieldKey FieldKey, termRange Ran
Prefix: fieldKey.Marshal(),
Reverse: reverse,
})
- field := Field{
+ field := index.Field{
Key: fieldKey,
Term: term,
}
@@ -120,7 +121,7 @@ func NewFieldIteratorTemplate(l *logger.Logger, fieldKey FieldKey, termRange Ran
if err != nil {
return nil, err
}
- return &FieldIteratorTemplate{
+ return &fieldIteratorTemplate{
delegated: newDelegateIterator(iter, fieldKey, l),
termRange: termRange,
fn: fn,
@@ -129,8 +130,8 @@ func NewFieldIteratorTemplate(l *logger.Logger, fieldKey FieldKey, termRange Ran
}, nil
}
-func parseKey(fieldKey FieldKey, key []byte) (Field, error) {
- f := &Field{
+func parseKey(fieldKey index.FieldKey, key []byte) (index.Field, error) {
+ f := &index.Field{
Key: fieldKey,
}
err := f.Unmarshal(key)
@@ -140,105 +141,18 @@ func parseKey(fieldKey FieldKey, key []byte) (Field, error) {
return *f, nil
}
-type SwitchFn = func(a, b []byte) bool
-
-var _ FieldIterator = (*mergedIterator)(nil)
-
-type mergedIterator struct {
- cur *PostingValue
- switchFn SwitchFn
- inner []FieldIterator
- drained []FieldIterator
- drainedCount int
- init bool
- closed bool
-}
-
-func NewMergedIterator(merged []FieldIterator, fn SwitchFn) FieldIterator {
- return &mergedIterator{
- inner: merged,
- drained: make([]FieldIterator, len(merged)),
- switchFn: fn,
- }
-}
-
-func (m *mergedIterator) Next() bool {
- if m.closed {
- return false
- }
- if m.allDrained() {
- return false
- }
- if !m.init {
- for i, iterator := range m.inner {
- if !iterator.Next() {
- m.drain(i)
- }
- }
- if m.allDrained() {
- return false
- }
- m.init = true
- }
- var head FieldIterator
- var headIndex int
- for i, iterator := range m.inner {
- if iterator == nil {
- continue
- }
- if head == nil {
- head = iterator
- continue
- }
- if m.switchFn(head.Val().Term, iterator.Val().Term) {
- head = iterator
- headIndex = i
- }
- }
- m.cur = head.Val()
- if !head.Next() {
- m.drain(headIndex)
- }
- return true
-}
-
-func (m *mergedIterator) Val() *PostingValue {
- return m.cur
-}
-
-func (m *mergedIterator) Close() error {
- m.closed = true
- var err error
- for _, iterator := range m.drained {
- if iterator == nil {
- continue
- }
- err = multierr.Append(err, iterator.Close())
- }
- return err
-}
-
-func (m *mergedIterator) drain(index int) {
- m.drained[index], m.inner[index] = m.inner[index], nil
- m.drainedCount++
-}
-
-func (m *mergedIterator) allDrained() bool {
- return m.drainedCount == len(m.inner)
-}
-
var _ kv.Iterator = (*delegateIterator)(nil)
type delegateIterator struct {
delegated kv.Iterator
l *logger.Logger
fieldKeyBytes []byte
- curField Field
- fieldKey FieldKey
+ curField index.Field
+ fieldKey index.FieldKey
closed bool
}
-func newDelegateIterator(delegated kv.Iterator, fieldKey FieldKey, l *logger.Logger) *delegateIterator {
+func newDelegateIterator(delegated kv.Iterator, fieldKey index.FieldKey, l *logger.Logger) *delegateIterator {
fieldKeyBytes := fieldKey.Marshal()
return &delegateIterator{
delegated: delegated,
@@ -268,7 +182,7 @@ func (di *delegateIterator) RawKey() []byte {
return di.delegated.RawKey()
}
-func (di *delegateIterator) Field() Field {
+func (di *delegateIterator) Field() index.Field {
return di.curField
}
diff --git a/pkg/index/lsm/lsm.go b/pkg/index/lsm/lsm.go
index dff5eaf..835a199 100644
--- a/pkg/index/lsm/lsm.go
+++ b/pkg/index/lsm/lsm.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package lsm implements a tree-based index repository.
package lsm
import (
@@ -56,12 +57,14 @@ func (s *store) Write(fields []index.Field, itemID common.ItemID) (err error) {
return err
}
+// StoreOpts wraps options to create the lsm repository.
type StoreOpts struct {
Logger *logger.Logger
Path string
MemTableSize int64
}
+// NewStore creates a new lsm index repository.
func NewStore(opts StoreOpts) (index.Store, error) {
var err error
var lsm kv.Store
diff --git a/pkg/index/lsm/search.go b/pkg/index/lsm/search.go
index 3f6f52e..4af241a 100644
--- a/pkg/index/lsm/search.go
+++ b/pkg/index/lsm/search.go
@@ -32,7 +32,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index/posting/roaring"
)
-var ErrUnsupportedOperation = errors.New("unsupported operation")
+var errUnsupportedOperation = errors.New("unsupported operation")
func (s *store) MatchField(fieldKey index.FieldKey) (list posting.List, err error) {
return s.Range(fieldKey, index.RangeOpts{})
@@ -49,7 +49,7 @@ func (s *store) MatchTerms(field index.Field) (list posting.List, err error) {
return nil
})
if errors.Is(err, kv.ErrKeyNotFound) {
- return roaring.EmptyPostingList, nil
+ return roaring.DummyPostingList, nil
}
return
}
@@ -57,7 +57,7 @@ func (s *store) MatchTerms(field index.Field) (list posting.List, err error) {
func (s *store) Range(fieldKey index.FieldKey, opts index.RangeOpts) (list posting.List, err error) {
iter, err := s.Iterator(fieldKey, opts, modelv1.Sort_SORT_ASC)
if err != nil {
- return roaring.EmptyPostingList, err
+ return roaring.DummyPostingList, err
}
list = roaring.NewPostingList()
for iter.Next() {
@@ -68,7 +68,7 @@ func (s *store) Range(fieldKey index.FieldKey, opts index.RangeOpts) (list posti
}
func (s *store) Iterator(fieldKey index.FieldKey, termRange index.RangeOpts, order modelv1.Sort) (index.FieldIterator, error) {
- return index.NewFieldIteratorTemplate(s.l, fieldKey, termRange, order, s.lsm,
+ return newFieldIteratorTemplate(s.l, fieldKey, termRange, order, s.lsm,
func(term, value []byte, delegated kv.Iterator) (*index.PostingValue, error) {
pv := &index.PostingValue{
Term: term,
@@ -98,5 +98,5 @@ func (s *store) Iterator(fieldKey index.FieldKey, termRange index.RangeOpts, ord
}
func (s *store) Match(_ index.FieldKey, _ []string) (posting.List, error) {
- return nil, errors.WithMessage(ErrUnsupportedOperation, "LSM-Tree index doesn't support full-text searching")
+ return nil, errors.WithMessage(errUnsupportedOperation, "LSM-Tree index doesn't support full-text searching")
}
diff --git a/pkg/index/posting/posting.go b/pkg/index/posting/posting.go
index 6f810fd..3807223 100644
--- a/pkg/index/posting/posting.go
+++ b/pkg/index/posting/posting.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package posting implements a posting list contains a list of document ids.
package posting
import (
@@ -23,6 +24,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
)
+// ErrListEmpty indicates the postings list is empty.
var ErrListEmpty = errors.New("postings list is empty")
// List is a collection of common.ItemID.
@@ -68,6 +70,7 @@ type List interface {
SizeInBytes() int64
}
+// Iterator allows iterating over a posting list.
type Iterator interface {
Next() bool
diff --git a/pkg/index/posting/roaring/roaring.go b/pkg/index/posting/roaring/roaring.go
index eaab6a3..3d31eb7 100644
--- a/pkg/index/posting/roaring/roaring.go
+++ b/pkg/index/posting/roaring/roaring.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package roaring implements the posting list by a roaring bitmap.
package roaring
import (
@@ -26,11 +27,12 @@ import (
)
var (
- EmptyPostingList = NewPostingList()
+ // DummyPostingList is an empty list.
+ DummyPostingList = NewPostingList()
- ErrIntersectRoaringOnly = errors.New("Intersect only supported between roaringDocId sets")
- ErrUnionRoaringOnly = errors.New("Union only supported between roaringDocId sets")
- ErrDifferenceRoaringOnly = errors.New("Difference only supported between roaringDocId sets")
+ errIntersectRoaringOnly = errors.New("Intersect only supported between roaringDocId sets")
+ errUnionRoaringOnly = errors.New("Union only supported between roaringDocId sets")
+ errDifferenceRoaringOnly = errors.New("Difference only supported between roaringDocId sets")
)
var _ posting.List = (*postingsList)(nil)
@@ -48,12 +50,14 @@ func (p *postingsList) Unmarshall(data []byte) error {
return p.bitmap.UnmarshalBinary(data)
}
+// NewPostingList returns a empty posting list.
func NewPostingList() posting.List {
return &postingsList{
bitmap: roaring64.New(),
}
}
+// NewPostingListWithInitialData return a posting list with initialized data.
func NewPostingListWithInitialData(data ...uint64) posting.List {
list := NewPostingList()
for _, d := range data {
@@ -62,6 +66,7 @@ func NewPostingListWithInitialData(data ...uint64) posting.List {
return list
}
+// NewRange return a posting list added the integers in [start, end).
func NewRange(start, end uint64) posting.List {
list := &postingsList{
bitmap: roaring64.New(),
@@ -128,7 +133,7 @@ func (p *postingsList) Insert(id common.ItemID) {
func (p *postingsList) Intersect(other posting.List) error {
o, ok := other.(*postingsList)
if !ok {
- return ErrIntersectRoaringOnly
+ return errIntersectRoaringOnly
}
p.bitmap.And(o.bitmap)
return nil
@@ -137,7 +142,7 @@ func (p *postingsList) Intersect(other posting.List) error {
func (p *postingsList) Difference(other posting.List) error {
o, ok := other.(*postingsList)
if !ok {
- return ErrDifferenceRoaringOnly
+ return errDifferenceRoaringOnly
}
p.bitmap.AndNot(o.bitmap)
return nil
@@ -146,7 +151,7 @@ func (p *postingsList) Difference(other posting.List) error {
func (p *postingsList) Union(other posting.List) error {
o, ok := other.(*postingsList)
if !ok {
- return ErrUnionRoaringOnly
+ return errUnionRoaringOnly
}
p.bitmap.Or(o.bitmap)
return nil
diff --git a/pkg/index/testcases/duration.go b/pkg/index/testcases/duration.go
index 8fd998d..a7221a1 100644
--- a/pkg/index/testcases/duration.go
+++ b/pkg/index/testcases/duration.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package testcases implements common helpers for testing inverted and lsm indices.
package testcases
import (
@@ -37,6 +38,7 @@ var duration = index.FieldKey{
IndexRuleID: 3,
}
+// SimpleStore is subset of index.Store for testing.
type SimpleStore interface {
index.FieldIterable
index.Writer
@@ -54,6 +56,7 @@ type result struct {
key int
}
+// RunDuration executes duration related cases.
func RunDuration(t *testing.T, data map[int]posting.List, store SimpleStore) {
tester := assert.New(t)
is := require.New(t)
@@ -309,6 +312,7 @@ func toArray(list posting.List) []int {
return ints
}
+// SetUpDuration initializes data for testing duration related cases.
func SetUpDuration(t *assert.Assertions, store index.Writer) map[int]posting.List {
r := map[int]posting.List{
50: roaring.NewPostingList(),
@@ -317,10 +321,10 @@ func SetUpDuration(t *assert.Assertions, store index.Writer) map[int]posting.Lis
1000: roaring.NewPostingList(),
2000: roaring.NewPostingList(),
}
- return SetUpPartialDuration(t, store, r)
+ return setUpPartialDuration(t, store, r)
}
-func SetUpPartialDuration(t *assert.Assertions, store index.Writer, r map[int]posting.List) map[int]posting.List {
+func setUpPartialDuration(t *assert.Assertions, store index.Writer, r map[int]posting.List) map[int]posting.List {
idx := make([]int, 0, len(r))
for key := range r {
idx = append(idx, key)
diff --git a/pkg/index/testcases/service_name.go b/pkg/index/testcases/service_name.go
index 9b024a0..a77020f 100644
--- a/pkg/index/testcases/service_name.go
+++ b/pkg/index/testcases/service_name.go
@@ -33,6 +33,7 @@ var serviceName = index.FieldKey{
IndexRuleID: 6,
}
+// RunServiceName executes service name related cases.
func RunServiceName(t *testing.T, store SimpleStore) {
tester := assert.New(t)
tests := []struct {
@@ -59,7 +60,7 @@ func RunServiceName(t *testing.T, store SimpleStore) {
},
{
name: "unknown field",
- want: roaring.EmptyPostingList,
+ want: roaring.DummyPostingList,
},
{
name: "unknown term",
@@ -67,7 +68,7 @@ func RunServiceName(t *testing.T, store SimpleStore) {
Key: serviceName,
Term: []byte("unknown"),
},
- want: roaring.EmptyPostingList,
+ want: roaring.DummyPostingList,
},
}
for _, tt := range tests {
@@ -84,6 +85,7 @@ func RunServiceName(t *testing.T, store SimpleStore) {
}
}
+// SetUp initializes a index repository.
func SetUp(t *assert.Assertions, store SimpleStore) {
for i := 0; i < 100; i++ {
if i < 100/2 {
diff --git a/pkg/iter/iter.go b/pkg/iter/iter.go
index d2406c6..233d24b 100644
--- a/pkg/iter/iter.go
+++ b/pkg/iter/iter.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package iter implement a generic Iterator.
package iter
// An Iterator is a stream of items of some type.
diff --git a/pkg/logger/jsonproto.go b/pkg/logger/jsonproto.go
index c1f8137..94fb025 100644
--- a/pkg/logger/jsonproto.go
+++ b/pkg/logger/jsonproto.go
@@ -24,10 +24,12 @@ import (
"google.golang.org/protobuf/proto"
)
+// MarshalError is the error raised by marshaling a JSON object.
type MarshalError struct {
Msg string `json:"msg"`
}
+// Proto converts proto.Message to JSON raw data.
func Proto(message proto.Message) []byte {
b, err := protojson.Marshal(message)
if err != nil {
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index 2b41ff7..57122b5 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -15,20 +15,19 @@
// specific language governing permissions and limitations
// under the License.
+// Package logger implements a logging system with a module tag.
+// The module tag represents a scope where the log event is emitted.
package logger
import (
"context"
"strings"
- "github.com/pkg/errors"
"github.com/rs/zerolog"
)
-var (
- ContextKey = contextKey{}
- ErrNoLoggerInContext = errors.New("no logger in context")
-)
+// ContextKey is the key to store Logger in the context.
+var ContextKey = contextKey{}
type contextKey struct{}
@@ -47,10 +46,12 @@ type Logger struct {
module string
}
+// Module returns logger's module name.
func (l Logger) Module() string {
return l.module
}
+// Named creates a new Logger and assigns a module to it.
func (l *Logger) Named(name ...string) *Logger {
var mm []string
if l.module == rootName {
@@ -80,19 +81,22 @@ type Loggable interface {
SetLogger(*Logger)
}
-func Fetch(ctx context.Context, name string) *Logger {
- return FetchOrDefault(ctx, name, nil)
+// Fetch gets a Logger in a context, then creates a new Logger based on it.
+func Fetch(ctx context.Context, newModuleName string) *Logger {
+ return FetchOrDefault(ctx, newModuleName, nil)
}
-func FetchOrDefault(ctx context.Context, name string, defaultLogger *Logger) *Logger {
+// FetchOrDefault gets a Logger in a context, then creates a new Logger based on it
+// If the context doesn't include a Logger. The default Logger will be picked.
+func FetchOrDefault(ctx context.Context, newModuleName string, defaultLogger *Logger) *Logger {
parentLogger := ctx.Value(ContextKey)
if parentLogger != nil {
if pl, ok := parentLogger.(*Logger); ok {
- return pl.Named(name)
+ return pl.Named(newModuleName)
}
}
if defaultLogger == nil {
- return GetLogger(name)
+ return GetLogger(newModuleName)
}
return defaultLogger
}
diff --git a/pkg/partition/entity.go b/pkg/partition/entity.go
index 24a358d..f5a8610 100644
--- a/pkg/partition/entity.go
+++ b/pkg/partition/entity.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// Package partition implements a location system to find a shard or index rule.
+// This system reflects the entity identity which is a intermediate result in calculating the target shard.
package partition
import (
@@ -27,15 +29,19 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
+// ErrMalformedElement indicates the element is malformed.
var ErrMalformedElement = errors.New("element is malformed")
+// EntityLocator combines several TagLocators that help find the entity value.
type EntityLocator []TagLocator
+// TagLocator contains offsets to retrieve a tag swiftly.
type TagLocator struct {
FamilyOffset int
TagOffset int
}
+// NewEntityLocator return a EntityLocator based on tag family spec and entity spec.
func NewEntityLocator(families []*databasev1.TagFamilySpec, entity *databasev1.Entity) EntityLocator {
locator := make(EntityLocator, 0, len(entity.GetTagNames()))
for _, tagInEntity := range entity.GetTagNames() {
@@ -47,6 +53,7 @@ func NewEntityLocator(families []*databasev1.TagFamilySpec, entity *databasev1.E
return locator
}
+// Find the entity from a tag family, prepend a subject to the entity.
func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite) (tsdb.Entity, tsdb.EntityValues, error) {
entityValues := make(tsdb.EntityValues, len(e)+1)
entityValues[0] = tsdb.StrValue(subject)
@@ -64,6 +71,7 @@ func (e EntityLocator) Find(subject string, value []*modelv1.TagFamilyForWrite)
return entity, entityValues, nil
}
+// Locate a shard and find the entity from a tag family, prepend a subject to the entity.
func (e EntityLocator) Locate(subject string, value []*modelv1.TagFamilyForWrite, shardNum uint32) (tsdb.Entity, tsdb.EntityValues, common.ShardID, error) {
entity, tagValues, err := e.Find(subject, value)
if err != nil {
@@ -76,6 +84,7 @@ func (e EntityLocator) Locate(subject string, value []*modelv1.TagFamilyForWrite
return entity, tagValues, common.ShardID(id), nil
}
+// GetTagByOffset gets a tag value based of a tag family offset and a tag offset in this family.
func GetTagByOffset(value []*modelv1.TagFamilyForWrite, fIndex, tIndex int) (*modelv1.TagValue, error) {
if fIndex >= len(value) {
return nil, errors.Wrap(ErrMalformedElement, "tag family offset is invalid")
diff --git a/pkg/partition/index.go b/pkg/partition/index.go
index 0cf38b3..f645178 100644
--- a/pkg/partition/index.go
+++ b/pkg/partition/index.go
@@ -22,11 +22,13 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
+// IndexRuleLocator combines several TagLocators that help find the index value.
type IndexRuleLocator struct {
Rule *databasev1.IndexRule
TagIndices []TagLocator
}
+// ParseIndexRuleLocators returns a IndexRuleLocator based on the tag family spec and index rules.
func ParseIndexRuleLocators(families []*databasev1.TagFamilySpec, indexRules []*databasev1.IndexRule) (locators []*IndexRuleLocator) {
for _, rule := range indexRules {
tagIndices := make([]TagLocator, 0, len(rule.GetTags()))
diff --git a/pkg/partition/route.go b/pkg/partition/route.go
index bcb18af..5a3b410 100644
--- a/pkg/partition/route.go
+++ b/pkg/partition/route.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/convert"
)
+// ShardID calculates a shard id.
func ShardID(key []byte, shardNum uint32) (uint, error) {
if shardNum < 1 {
return 0, errors.New("invalid shardNum")
diff --git a/pkg/pb/v1/metadata.go b/pkg/pb/v1/metadata.go
index 13b3202..7de4753 100644
--- a/pkg/pb/v1/metadata.go
+++ b/pkg/pb/v1/metadata.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package v1 implements helpers to access data defined by API v1.
package v1
import (
@@ -22,6 +23,8 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
+// FindTagByName finds TagSpec in several tag families by its name.
+// The tag name should be unique in these families.
func FindTagByName(families []*databasev1.TagFamilySpec, tagName string) (int, int, *databasev1.TagSpec) {
for fi, family := range families {
for ti, tag := range family.Tags {
@@ -33,7 +36,7 @@ func FindTagByName(families []*databasev1.TagFamilySpec, tagName string) (int, i
return 0, 0, nil
}
-func TagValueTypeConv(tagValue *modelv1.TagValue) (tagType databasev1.TagType, isNull bool) {
+func tagValueTypeConv(tagValue *modelv1.TagValue) (tagType databasev1.TagType, isNull bool) {
switch tagValue.GetValue().(type) {
case *modelv1.TagValue_Int:
return databasev1.TagType_TAG_TYPE_INT, false
@@ -53,6 +56,7 @@ func TagValueTypeConv(tagValue *modelv1.TagValue) (tagType databasev1.TagType, i
return databasev1.TagType_TAG_TYPE_UNSPECIFIED, false
}
+// FieldValueTypeConv recognizes the field type from its value.
func FieldValueTypeConv(tagValue *modelv1.FieldValue) (tagType databasev1.FieldType, isNull bool) {
switch tagValue.GetValue().(type) {
case *modelv1.FieldValue_Int:
@@ -67,6 +71,7 @@ func FieldValueTypeConv(tagValue *modelv1.FieldValue) (tagType databasev1.FieldT
return databasev1.FieldType_FIELD_TYPE_UNSPECIFIED, false
}
+// ParseMaxModRevision parses the index rule's max revision field from its metadata.
func ParseMaxModRevision(indexRules []*databasev1.IndexRule) (maxRevisionForIdxRules int64) {
maxRevisionForIdxRules = int64(0)
for _, idxRule := range indexRules {
diff --git a/pkg/pb/v1/tsdb/interval.go b/pkg/pb/v1/tsdb/interval.go
index 0019645..75f9f83 100644
--- a/pkg/pb/v1/tsdb/interval.go
+++ b/pkg/pb/v1/tsdb/interval.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package tsdb implements helpers around tsdb.IntervalRule.
package tsdb
import (
@@ -24,8 +25,9 @@ import (
"github.com/apache/skywalking-banyandb/banyand/tsdb"
)
-var ErrInvalidUnit = errors.New("invalid interval rule's unit")
+var errInvalidUnit = errors.New("invalid interval rule's unit")
+// ToIntervalRule decodes commonv1.IntervalRule to tsdb.IntervalRule.
func ToIntervalRule(ir *commonv1.IntervalRule) (result tsdb.IntervalRule, err error) {
switch ir.Unit {
case commonv1.IntervalRule_UNIT_DAY:
@@ -33,7 +35,7 @@ func ToIntervalRule(ir *commonv1.IntervalRule) (result tsdb.IntervalRule, err er
case commonv1.IntervalRule_UNIT_HOUR:
result.Unit = tsdb.HOUR
default:
- return result, ErrInvalidUnit
+ return result, errInvalidUnit
}
result.Num = int(ir.Num)
return result, err
diff --git a/pkg/pb/v1/write.go b/pkg/pb/v1/write.go
index 7434966..03c470b 100644
--- a/pkg/pb/v1/write.go
+++ b/pkg/pb/v1/write.go
@@ -24,33 +24,30 @@ import (
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/timestamppb"
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/convert"
)
-type ID string
-
const fieldFlagLength = 9
var zeroFieldValue = &modelv1.FieldValue{Value: &modelv1.FieldValue_Int{Int: &modelv1.Int{Value: 0}}}
var (
+ // TagFlag is a flag suffix to identify the encoding method.
+ TagFlag = make([]byte, fieldFlagLength)
+
strDelimiter = []byte("\n")
- NullTag = &modelv1.TagValue{Value: &modelv1.TagValue_Null{}}
- TagFlag = make([]byte, fieldFlagLength)
+ nullTag = &modelv1.TagValue{Value: &modelv1.TagValue_Null{}}
+ nullTagValue = TagValue{}
- ErrUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
- ErrNullValue = errors.New("the tag value is null")
- ErrMalformedElement = errors.New("element is malformed")
- ErrMalformedField = errors.New("field is malformed")
+ errUnsupportedTagForIndexField = errors.New("the tag type(for example, null) can not be as the index field value")
+ errMalformedElement = errors.New("element is malformed")
+ errMalformedField = errors.New("field is malformed")
)
+// MarshalTagValue encodes modelv1.TagValue to bytes.
func MarshalTagValue(tagValue *modelv1.TagValue) ([]byte, error) {
fv, err := ParseTagValue(tagValue)
if err != nil {
@@ -63,12 +60,29 @@ func MarshalTagValue(tagValue *modelv1.TagValue) ([]byte, error) {
return fv.marshalArr(), nil
}
+// TagValue seels single value and array value.
type TagValue struct {
value []byte
arr [][]byte
splitter []byte
}
+// GetValue returns the single value.
+func (fv TagValue) GetValue() []byte {
+ if len(fv.value) < 1 {
+ return nil
+ }
+ return fv.value
+}
+
+// GetArr returns the array value.
+func (fv TagValue) GetArr() [][]byte {
+ if len(fv.arr) < 1 {
+ return nil
+ }
+ return fv.arr
+}
+
func newValue(value []byte) TagValue {
return TagValue{
value: value,
@@ -89,20 +103,6 @@ func appendValue(fv *TagValue, value []byte) *TagValue {
return fv
}
-func (fv TagValue) GetValue() []byte {
- if len(fv.value) < 1 {
- return nil
- }
- return fv.value
-}
-
-func (fv TagValue) GetArr() [][]byte {
- if len(fv.arr) < 1 {
- return nil
- }
- return fv.arr
-}
-
func (fv *TagValue) marshalArr() []byte {
switch len(fv.arr) {
case 0:
@@ -126,10 +126,11 @@ func (fv *TagValue) marshalArr() []byte {
return buf.Bytes()
}
+// ParseTagValue decodes modelv1.TagValue to TagValue.
func ParseTagValue(tagValue *modelv1.TagValue) (TagValue, error) {
switch x := tagValue.GetValue().(type) {
case *modelv1.TagValue_Null:
- return TagValue{}, ErrNullValue
+ return nullTagValue, nil
case *modelv1.TagValue_Str:
return newValue([]byte(x.Str.GetValue())), nil
case *modelv1.TagValue_Int:
@@ -151,215 +152,23 @@ func ParseTagValue(tagValue *modelv1.TagValue) (TagValue, error) {
case *modelv1.TagValue_Id:
return newValue([]byte(x.Id.GetValue())), nil
}
- return TagValue{}, ErrUnsupportedTagForIndexField
-}
-
-type StreamWriteRequestBuilder struct {
- ec *streamv1.WriteRequest
-}
-
-func NewStreamWriteRequestBuilder() *StreamWriteRequestBuilder {
- return &StreamWriteRequestBuilder{
- ec: &streamv1.WriteRequest{
- Element: &streamv1.ElementValue{
- TagFamilies: make([]*modelv1.TagFamilyForWrite, 0),
- },
- },
- }
-}
-
-func (b *StreamWriteRequestBuilder) Metadata(group, name string) *StreamWriteRequestBuilder {
- b.ec.Metadata = &commonv1.Metadata{
- Group: group,
- Name: name,
- }
- return b
-}
-
-func (b *StreamWriteRequestBuilder) ID(id string) *StreamWriteRequestBuilder {
- b.ec.Element.ElementId = id
- return b
-}
-
-func (b *StreamWriteRequestBuilder) Timestamp(t time.Time) *StreamWriteRequestBuilder {
- b.ec.Element.Timestamp = timestamppb.New(t)
- return b
-}
-
-func (b *StreamWriteRequestBuilder) TagFamily(tags ...interface{}) *StreamWriteRequestBuilder {
- tagFamily := &modelv1.TagFamilyForWrite{}
- for _, tag := range tags {
- tagFamily.Tags = append(tagFamily.Tags, getTag(tag))
- }
- b.ec.Element.TagFamilies = append(b.ec.Element.TagFamilies, tagFamily)
- return b
-}
-
-func (b *StreamWriteRequestBuilder) Build() *streamv1.WriteRequest {
- return b.ec
-}
-
-func getTag(tag interface{}) *modelv1.TagValue {
- if tag == nil {
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Null{},
- }
- }
- switch t := tag.(type) {
- case int:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Int{
- Int: &modelv1.Int{
- Value: int64(t),
- },
- },
- }
- case string:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: t,
- },
- },
- }
- case []byte:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_BinaryData{
- BinaryData: t,
- },
- }
- case ID:
- return &modelv1.TagValue{
- Value: &modelv1.TagValue_Id{
- Id: &modelv1.ID{
- Value: string(t),
- },
- },
- }
- }
- return nil
-}
-
-type MeasureWriteRequestBuilder struct {
- ec *measurev1.WriteRequest
-}
-
-func NewMeasureWriteRequestBuilder() *MeasureWriteRequestBuilder {
- return &MeasureWriteRequestBuilder{
- ec: &measurev1.WriteRequest{
- DataPoint: &measurev1.DataPointValue{
- TagFamilies: make([]*modelv1.TagFamilyForWrite, 0),
- Fields: make([]*modelv1.FieldValue, 0),
- },
- },
- }
-}
-
-func (b *MeasureWriteRequestBuilder) Metadata(group, name string) *MeasureWriteRequestBuilder {
- b.ec.Metadata = &commonv1.Metadata{
- Group: group,
- Name: name,
- }
- return b
-}
-
-func (b *MeasureWriteRequestBuilder) TagFamily(tags ...interface{}) *MeasureWriteRequestBuilder {
- tagFamily := &modelv1.TagFamilyForWrite{}
- for _, tag := range tags {
- tagFamily.Tags = append(tagFamily.Tags, getTag(tag))
- }
- b.ec.DataPoint.TagFamilies = append(b.ec.DataPoint.TagFamilies, tagFamily)
- return b
-}
-
-func (b *MeasureWriteRequestBuilder) Fields(fields ...interface{}) *MeasureWriteRequestBuilder {
- fieldValues := make([]*modelv1.FieldValue, 0)
- for _, field := range fields {
- fieldValues = append(fieldValues, getField(field))
- }
- b.ec.DataPoint.Fields = append(b.ec.DataPoint.Fields, fieldValues...)
- return b
-}
-
-func (b *MeasureWriteRequestBuilder) Timestamp(t time.Time) *MeasureWriteRequestBuilder {
- b.ec.DataPoint.Timestamp = timestamppb.New(t)
- return b
-}
-
-func (b *MeasureWriteRequestBuilder) Build() *measurev1.WriteRequest {
- return b.ec
-}
-
-func getField(field interface{}) *modelv1.FieldValue {
- if field == nil {
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Null{},
- }
- }
- switch t := field.(type) {
- case int8:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: int64(t),
- },
- },
- }
- case int32:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: int64(t),
- },
- },
- }
- case int64:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: t,
- },
- },
- }
- case int:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: int64(t),
- },
- },
- }
- case string:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_Str{
- Str: &modelv1.Str{
- Value: t,
- },
- },
- }
- case []byte:
- return &modelv1.FieldValue{
- Value: &modelv1.FieldValue_BinaryData{
- BinaryData: t,
- },
- }
- }
- return nil
+ return TagValue{}, errUnsupportedTagForIndexField
}
+// EncodeFamily encodes a tag family to bytes by referring to its specification.
func EncodeFamily(familySpec *databasev1.TagFamilySpec, family *modelv1.TagFamilyForWrite) ([]byte, error) {
if len(family.GetTags()) > len(familySpec.GetTags()) {
- return nil, errors.Wrap(ErrMalformedElement, "tag number is more than expected")
+ return nil, errors.Wrap(errMalformedElement, "tag number is more than expected")
}
data := &modelv1.TagFamilyForWrite{}
for ti, tag := range family.GetTags() {
tagSpec := familySpec.GetTags()[ti]
- tType, isNull := TagValueTypeConv(tag)
+ tType, isNull := tagValueTypeConv(tag)
if !isNull && tType != tagSpec.GetType() {
- return nil, errors.Wrapf(ErrMalformedElement, "tag %s type is unexpected", tagSpec.GetName())
+ return nil, errors.Wrapf(errMalformedElement, "tag %s type is unexpected", tagSpec.GetName())
}
if tagSpec.IndexedOnly {
- data.Tags = append(data.Tags, NullTag)
+ data.Tags = append(data.Tags, nullTag)
} else {
data.Tags = append(data.Tags, tag)
}
@@ -367,6 +176,7 @@ func EncodeFamily(familySpec *databasev1.TagFamilySpec, family *modelv1.TagFamil
return proto.Marshal(data)
}
+// DecodeFieldValue decodes bytes to field value based on its specification.
func DecodeFieldValue(fieldValue []byte, fieldSpec *databasev1.FieldSpec) (*modelv1.FieldValue, error) {
switch fieldSpec.GetFieldType() {
case databasev1.FieldType_FIELD_TYPE_STRING:
@@ -376,7 +186,7 @@ func DecodeFieldValue(fieldValue []byte, fieldSpec *databasev1.FieldSpec) (*mode
return zeroFieldValue, nil
}
if len(fieldValue) != 8 {
- return nil, errors.WithMessagef(ErrMalformedField, "the length of encoded field value(int64) %s is %d, less than 8",
+ return nil, errors.WithMessagef(errMalformedField, "the length of encoded field value(int64) %s is %d, less than 8",
hex.EncodeToString(fieldValue), len(fieldValue))
}
return &modelv1.FieldValue{Value: &modelv1.FieldValue_Int{Int: &modelv1.Int{Value: convert.BytesToInt64(fieldValue)}}}, nil
@@ -386,6 +196,7 @@ func DecodeFieldValue(fieldValue []byte, fieldSpec *databasev1.FieldSpec) (*mode
return &modelv1.FieldValue{Value: &modelv1.FieldValue_Null{}}, nil
}
+// EncoderFieldFlag encodes the encoding method, compression method, and interval into bytes.
func EncoderFieldFlag(fieldSpec *databasev1.FieldSpec, interval time.Duration) []byte {
encodingMethod := byte(fieldSpec.GetEncodingMethod().Number())
compressionMethod := byte(fieldSpec.GetCompressionMethod().Number())
@@ -395,9 +206,10 @@ func EncoderFieldFlag(fieldSpec *databasev1.FieldSpec, interval time.Duration) [
return bb
}
+// DecodeFieldFlag decodes the encoding method, compression method, and interval from bytes.
func DecodeFieldFlag(key []byte) (*databasev1.FieldSpec, time.Duration, error) {
if len(key) < fieldFlagLength {
- return nil, 0, errors.WithMessagef(ErrMalformedField, "flag %s is invalid", hex.EncodeToString(key))
+ return nil, 0, errors.WithMessagef(errMalformedField, "flag %s is invalid", hex.EncodeToString(key))
}
b := key[len(key)-9:]
return &databasev1.FieldSpec{
diff --git a/pkg/query/aggregation/aggregation.go b/pkg/query/aggregation/aggregation.go
index 69d9320..a0836f9 100644
--- a/pkg/query/aggregation/aggregation.go
+++ b/pkg/query/aggregation/aggregation.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package aggregation implements aggregation functions to statistic a range of values.
package aggregation
import (
@@ -25,14 +26,16 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
-var ErrUnknownFunc = errors.New("unknown aggregation function")
+var errUnknownFunc = errors.New("unknown aggregation function")
+// Int64Func allows to aggregate int64.
type Int64Func interface {
In(int64)
Val() int64
Reset()
}
+// NewInt64Func returns a Int64Func based on function type.
func NewInt64Func(af modelv1.AggregationFunction) (Int64Func, error) {
switch af {
case modelv1.AggregationFunction_AGGREGATION_FUNCTION_MEAN:
@@ -50,5 +53,5 @@ func NewInt64Func(af modelv1.AggregationFunction) (Int64Func, error) {
case modelv1.AggregationFunction_AGGREGATION_FUNCTION_SUM:
return &sumInt64Func{}, nil
}
- return nil, ErrUnknownFunc
+ return nil, errUnknownFunc
}
diff --git a/pkg/query/executor/interface.go b/pkg/query/executor/interface.go
index 9382542..b5b184a 100644
--- a/pkg/query/executor/interface.go
+++ b/pkg/query/executor/interface.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package executor defines the specifications accessing underlying data repositories.
package executor
import (
@@ -25,26 +26,31 @@ import (
"github.com/apache/skywalking-banyandb/banyand/tsdb"
)
+// ExecutionContext allows retrieving data from tsdb.
type ExecutionContext interface {
Shards(entity tsdb.Entity) ([]tsdb.Shard, error)
Shard(id common.ShardID) (tsdb.Shard, error)
ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFamily, error)
}
+// StreamExecutionContext allows retrieving data through the stream module.
type StreamExecutionContext interface {
ExecutionContext
ParseElementID(item tsdb.Item) (string, error)
}
+// StreamExecutable allows querying in the stream schema.
type StreamExecutable interface {
Execute(StreamExecutionContext) ([]*streamv1.Element, error)
}
+// MeasureExecutionContext allows retrieving data through the measure module.
type MeasureExecutionContext interface {
ExecutionContext
ParseField(name string, item tsdb.Item) (*measurev1.DataPoint_Field, error)
}
+// MIterator allows iterating in a measure data set.
type MIterator interface {
Next() bool
@@ -53,22 +59,7 @@ type MIterator interface {
Close() error
}
-var EmptyMIterator = emptyMIterator{}
-
-type emptyMIterator struct{}
-
-func (ei emptyMIterator) Next() bool {
- return false
-}
-
-func (ei emptyMIterator) Current() []*measurev1.DataPoint {
- return nil
-}
-
-func (ei emptyMIterator) Close() error {
- return nil
-}
-
+// MeasureExecutable allows querying in the measure schema.
type MeasureExecutable interface {
Execute(MeasureExecutionContext) (MIterator, error)
}
diff --git a/pkg/query/logical/common.go b/pkg/query/logical/common.go
index 92ceef7..8ddb295 100644
--- a/pkg/query/logical/common.go
+++ b/pkg/query/logical/common.go
@@ -33,25 +33,25 @@ import (
)
var (
- ErrTagNotDefined = errors.New("tag is not defined")
- ErrFieldNotDefined = errors.New("field is not defined")
- ErrUnsupportedConditionOp = errors.New("unsupported condition operation")
- ErrUnsupportedConditionValue = errors.New("unsupported condition value type")
- ErrInvalidCriteriaType = errors.New("invalid criteria type")
- ErrIncompatibleQueryCondition = errors.New("incompatible query condition type")
- ErrIndexNotDefined = errors.New("index is not define for the tag")
- ErrMultipleGlobalIndexes = errors.New("multiple global indexes are not supported")
- ErrInvalidData = errors.New("data is invalid")
+ errTagNotDefined = errors.New("tag is not defined")
+ errUnsupportedConditionOp = errors.New("unsupported condition operation")
+ errUnsupportedConditionValue = errors.New("unsupported condition value type")
+ errInvalidCriteriaType = errors.New("invalid criteria type")
+ errIndexNotDefined = errors.New("index is not define for the tag")
+ errInvalidData = errors.New("data is invalid")
nullTag = &modelv1.TagValue{Value: &modelv1.TagValue_Null{}}
)
type (
+ // SeekerBuilder wraps the execution of tsdb.SeekerBuilder.
+ // TODO:// we could have a chance to remove this wrapper.
SeekerBuilder func(builder tsdb.SeekerBuilder)
- Comparator func(a, b tsdb.Item) bool
+
+ comparator func(a, b tsdb.Item) bool
)
-func CreateComparator(sortDirection modelv1.Sort) Comparator {
+func createComparator(sortDirection modelv1.Sort) comparator {
return func(a, b tsdb.Item) bool {
comp := bytes.Compare(a.SortedField(), b.SortedField())
if sortDirection == modelv1.Sort_SORT_DESC {
@@ -72,13 +72,13 @@ func ProjectItem(ec executor.ExecutionContext, item tsdb.Item, projectionFieldRe
continue
}
tags := make([]*modelv1.Tag, len(refs))
- familyName := refs[0].Tag.GetFamilyName()
+ familyName := refs[0].Tag.getFamilyName()
parsedTagFamily, err := ec.ParseTagFamily(familyName, item)
if err != nil {
return nil, errors.WithMessage(err, "parse projection")
}
if len(refs) > len(parsedTagFamily.Tags) {
- return nil, errors.Wrapf(ErrInvalidData,
+ return nil, errors.Wrapf(errInvalidData,
"the number of tags %d in %s is less then expected %d",
len(parsedTagFamily.Tags), familyName, len(refs))
}
@@ -144,12 +144,16 @@ func ExecuteForShard(l *logger.Logger, series tsdb.SeriesList, timeRange timesta
return itersInShard, closers, nil
}
+// DefaultLimit is the default limit size of a querying.
var DefaultLimit uint32 = 20
+// Tag represents the combination of tag family and tag name.
+// It's a tag's identity.
type Tag struct {
familyName, name string
}
+// NewTag return a new Tag.
func NewTag(family, name string) *Tag {
return &Tag{
familyName: family,
@@ -171,22 +175,26 @@ func (t *Tag) GetCompoundName() string {
return t.familyName + ":" + t.name
}
-func (t *Tag) GetTagName() string {
+func (t *Tag) getTagName() string {
return t.name
}
-func (t *Tag) GetFamilyName() string {
+func (t *Tag) getFamilyName() string {
return t.familyName
}
+// Field identity a field in a measure.
type Field struct {
Name string
}
+// NewField return a new Field.
func NewField(name string) *Field {
return &Field{Name: name}
}
+// StringSlicesEqual reports whether a and b are the same length and contain the same strings.
+// A nil argument is equivalent to an empty slice.
func StringSlicesEqual(a, b []string) bool {
if len(a) != len(b) {
return false
diff --git a/pkg/convert/hash.go b/pkg/query/logical/doc.go
similarity index 80%
copy from pkg/convert/hash.go
copy to pkg/query/logical/doc.go
index b08c2a5..9921331 100644
--- a/pkg/convert/hash.go
+++ b/pkg/query/logical/doc.go
@@ -15,14 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package convert
-
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
+// Package logical implements the executable operations.
+// It also provides Analyzers to parse logical expressions to operations,
+// and wire them to executable tree or execution plan.
+package logical
diff --git a/pkg/query/logical/expr.go b/pkg/query/logical/expr.go
index 0775855..b750dcd 100644
--- a/pkg/query/logical/expr.go
+++ b/pkg/query/logical/expr.go
@@ -23,9 +23,9 @@ import (
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
)
-var _ ResolvableExpr = (*TagRef)(nil)
+var _ Expr = (*TagRef)(nil)
-// TagRef is the reference to the field
+// TagRef is the reference to the tag
// also it holds the definition (derived from the streamSchema, measureSchema) of the tag.
type TagRef struct {
// Tag defines the family name and name of the Tag
@@ -34,13 +34,15 @@ type TagRef struct {
Spec *TagSpec
}
+// Equal reports whether f and expr have the same name and data type.
func (f *TagRef) Equal(expr Expr) bool {
if other, ok := expr.(*TagRef); ok {
- return other.Tag.GetTagName() == f.Tag.GetTagName() && other.Spec.Spec.GetType() == f.Spec.Spec.GetType()
+ return other.Tag.getTagName() == f.Tag.getTagName() && other.Spec.Spec.GetType() == f.Spec.Spec.GetType()
}
return false
}
+// DataType shows the type of the tag's value.
func (f *TagRef) DataType() int32 {
if f.Spec == nil {
panic("should be resolved first")
@@ -48,19 +50,12 @@ func (f *TagRef) DataType() int32 {
return int32(f.Spec.Spec.GetType())
}
-func (f *TagRef) Resolve(s Schema) error {
- specs, err := s.CreateTagRef([]*Tag{f.Tag})
- if err != nil {
- return err
- }
- f.Spec = specs[0][0].Spec
- return nil
-}
-
+// String shows the string representation.
func (f *TagRef) String() string {
return fmt.Sprintf("#%s<%s>", f.Tag.GetCompoundName(), f.Spec.Spec.GetType().String())
}
+// NewTagRef returns a new TagRef.
func NewTagRef(familyName, tagName string) *TagRef {
return &TagRef{
Tag: NewTag(familyName, tagName),
@@ -74,7 +69,7 @@ func NewSearchableTagRef(tagName string) *TagRef {
}
}
-var _ ResolvableExpr = (*FieldRef)(nil)
+var _ Expr = (*FieldRef)(nil)
// FieldRef is the reference to the field
// also it holds the definition (derived from measureSchema) of the field.
@@ -85,10 +80,12 @@ type FieldRef struct {
Spec *FieldSpec
}
+// String shows the string representation.
func (f *FieldRef) String() string {
return fmt.Sprintf("#%s<%s>", f.Spec.Spec.GetName(), f.Spec.Spec.GetFieldType().String())
}
+// DataType shows the type of the filed's value.
func (f *FieldRef) DataType() int32 {
if f.Spec == nil {
panic("should be resolved first")
@@ -96,6 +93,7 @@ func (f *FieldRef) DataType() int32 {
return int32(f.Spec.Spec.GetFieldType())
}
+// Equal reports whether f and expr have the same name and data type.
func (f *FieldRef) Equal(expr Expr) bool {
if other, ok := expr.(*FieldRef); ok {
return other.Field.Name == f.Field.Name && other.Spec.Spec.GetFieldType() == f.Spec.Spec.GetFieldType()
@@ -103,15 +101,8 @@ func (f *FieldRef) Equal(expr Expr) bool {
return false
}
-func (f *FieldRef) Resolve(s Schema) error {
- specs, err := s.CreateFieldRef(f.Field)
- if err != nil {
- return err
- }
- f.Spec = specs[0].Spec
- return nil
-}
-
+// FieldSpec is the reference to the field.
+// It also holds the definition of the field.
type FieldSpec struct {
Spec *databasev1.FieldSpec
FieldIdx int
diff --git a/pkg/query/logical/expr_literal.go b/pkg/query/logical/expr_literal.go
index 0b8bc89..b4ebb62 100644
--- a/pkg/query/logical/expr_literal.go
+++ b/pkg/query/logical/expr_literal.go
@@ -80,10 +80,6 @@ func (i *int64Literal) Equal(expr Expr) bool {
return false
}
-func Int(num int64) Expr {
- return &int64Literal{num}
-}
-
func (i *int64Literal) DataType() int32 {
return int32(databasev1.TagType_TAG_TYPE_INT)
}
@@ -157,12 +153,6 @@ func (i *int64ArrLiteral) Equal(expr Expr) bool {
return false
}
-func Ints(ints ...int64) Expr {
- return &int64ArrLiteral{
- arr: ints,
- }
-}
-
func (i *int64ArrLiteral) DataType() int32 {
return int32(databasev1.TagType_TAG_TYPE_INT_ARRAY)
}
@@ -221,7 +211,7 @@ func (s *strLiteral) Equal(expr Expr) bool {
return false
}
-func Str(str string) LiteralExpr {
+func str(str string) LiteralExpr {
return &strLiteral{str}
}
@@ -298,12 +288,6 @@ func (s *strArrLiteral) Equal(expr Expr) bool {
return false
}
-func Strs(strs ...string) Expr {
- return &strArrLiteral{
- arr: strs,
- }
-}
-
func (s *strArrLiteral) DataType() int32 {
return int32(databasev1.TagType_TAG_TYPE_STRING_ARRAY)
}
@@ -365,7 +349,7 @@ func (s *idLiteral) Equal(expr Expr) bool {
return false
}
-func ID(id string) LiteralExpr {
+func id(id string) LiteralExpr {
return &idLiteral{id}
}
@@ -415,15 +399,15 @@ var (
type nullLiteral struct{}
-func (s nullLiteral) Compare(other LiteralExpr) (int, bool) {
+func (s nullLiteral) Compare(_ LiteralExpr) (int, bool) {
return 0, false
}
-func (s nullLiteral) BelongTo(other LiteralExpr) bool {
+func (s nullLiteral) BelongTo(_ LiteralExpr) bool {
return false
}
-func (s nullLiteral) Contains(other LiteralExpr) bool {
+func (s nullLiteral) Contains(_ LiteralExpr) bool {
return false
}
@@ -431,7 +415,7 @@ func (s nullLiteral) Bytes() [][]byte {
return nil
}
-func (s nullLiteral) Equal(expr Expr) bool {
+func (s nullLiteral) Equal(_ Expr) bool {
return false
}
diff --git a/pkg/query/logical/format.go b/pkg/query/logical/format.go
index 6cc9b8f..8be9faf 100644
--- a/pkg/query/logical/format.go
+++ b/pkg/query/logical/format.go
@@ -21,26 +21,7 @@ import (
"strings"
)
-func Format(p Plan) string {
- return formatWithIndent(p, 0)
-}
-
-func formatWithIndent(p Plan, indent int) string {
- res := ""
- if indent > 1 {
- res += strings.Repeat(" ", 5*(indent-1))
- }
- if indent > 0 {
- res += "+"
- res += strings.Repeat("-", 4)
- }
- res += p.String() + "\n"
- for _, child := range p.Children() {
- res += formatWithIndent(child, indent+1)
- }
- return res
-}
-
+// FormatTagRefs outputs formatted tag refs.
func FormatTagRefs(sep string, exprGroup ...[]*TagRef) string {
var exprsStr []string
for _, exprs := range exprGroup {
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index 604161b..6149f96 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -34,12 +34,10 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index/posting"
)
-var (
- ErrNotRangeOperation = errors.New("this is not an range operation")
- ErrEmptyTree = errors.New("tree is empty")
- ErrInvalidLogicalExpression = errors.New("invalid logical expression")
-)
+var errInvalidLogicalExpression = errors.New("invalid logical expression")
+// GlobalIndexError represents a index rule is "global".
+// The local filter can't handle it.
type GlobalIndexError struct {
IndexRule *databasev1.IndexRule
Expr LiteralExpr
@@ -47,6 +45,8 @@ type GlobalIndexError struct {
func (g GlobalIndexError) Error() string { return g.IndexRule.String() }
+// BuildLocalFilter returns a new index.Filter for local indices.
+// It could parse series Path at the same time.
func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[string]int, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
if criteria == nil {
return nil, []tsdb.Entity{entity}, nil
@@ -63,7 +63,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
- return nil, nil, &GlobalIndexError{
+ return nil, nil, GlobalIndexError{
IndexRule: indexRule,
Expr: expr,
}
@@ -74,7 +74,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
case *modelv1.Criteria_Le:
le := criteria.GetLe()
if le.GetLeft() == nil && le.GetRight() == nil {
- return nil, nil, errors.WithMessagef(ErrInvalidLogicalExpression, "both sides(left and right) of [%v] are empty", criteria)
+ return nil, nil, errors.WithMessagef(errInvalidLogicalExpression, "both sides(left and right) of [%v] are empty", criteria)
}
left, leftEntities, err := BuildLocalFilter(le.Left, schema, entityDict, entity)
if err != nil {
@@ -102,7 +102,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
return or, entities, nil
}
}
- return nil, nil, ErrInvalidCriteriaType
+ return nil, nil, errInvalidCriteriaType
}
func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, expr LiteralExpr, entity tsdb.Entity) (index.Filter, []tsdb.Entity, error) {
@@ -150,7 +150,7 @@ func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, ex
case modelv1.Condition_BINARY_OP_NOT_IN:
panic("unimplemented")
}
- return nil, nil, errors.WithMessagef(ErrUnsupportedConditionOp, "index filter parses %v", cond)
+ return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "index filter parses %v", cond)
}
func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, tsdb.Entity, error) {
@@ -158,7 +158,7 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
copy(parsedEntity, entity)
entityIdx, ok := entityDict[cond.Name]
if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ {
- return nil, nil, errors.WithMessagef(ErrUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
+ return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
}
switch v := cond.Value.Value.(type) {
case *modelv1.TagValue_Str:
@@ -166,13 +166,13 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
parsedEntity[entityIdx] = []byte(v.Str.GetValue())
return nil, parsedEntity, nil
}
- return Str(v.Str.GetValue()), nil, nil
+ return str(v.Str.GetValue()), nil, nil
case *modelv1.TagValue_Id:
if ok {
parsedEntity[entityIdx] = []byte(v.Id.GetValue())
return nil, parsedEntity, nil
}
- return ID(v.Id.GetValue()), nil, nil
+ return id(v.Id.GetValue()), nil, nil
case *modelv1.TagValue_StrArray:
return &strArrLiteral{
@@ -193,7 +193,7 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
case *modelv1.TagValue_Null:
return nullLiteralExpr, nil, nil
}
- return nil, nil, errors.WithMessagef(ErrUnsupportedConditionValue, "index filter parses %v", cond)
+ return nil, nil, errors.WithMessagef(errUnsupportedConditionValue, "index filter parses %v", cond)
}
func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
@@ -236,15 +236,15 @@ func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, le
return []tsdb.Entity{result}
}
-type FieldKey struct {
+type fieldKey struct {
*databasev1.IndexRule
}
-func newFieldKey(indexRule *databasev1.IndexRule) FieldKey {
- return FieldKey{indexRule}
+func newFieldKey(indexRule *databasev1.IndexRule) fieldKey {
+ return fieldKey{indexRule}
}
-func (fk FieldKey) ToIndex(seriesID common.SeriesID) index.FieldKey {
+func (fk fieldKey) toIndex(seriesID common.SeriesID) index.FieldKey {
return index.FieldKey{
IndexRuleID: fk.Metadata.Id,
Analyzer: fk.Analyzer,
@@ -381,7 +381,7 @@ func (on *orNode) String() string {
type leaf struct {
index.Filter
- Key FieldKey
+ Key fieldKey
Expr LiteralExpr
}
@@ -394,7 +394,7 @@ func (l *leaf) MarshalJSON() ([]byte, error) {
type not struct {
index.Filter
- Key FieldKey
+ Key fieldKey
Inner index.Filter
}
@@ -410,7 +410,7 @@ func (n *not) Execute(searcher index.GetSearcher, seriesID common.SeriesID) (pos
if err != nil {
return nil, err
}
- all, err := s.MatchField(n.Key.ToIndex(seriesID))
+ all, err := s.MatchField(n.Key.toIndex(seriesID))
if err != nil {
return nil, err
}
@@ -451,7 +451,7 @@ func (eq *eq) Execute(searcher index.GetSearcher, seriesID common.SeriesID) (pos
return nil, err
}
return s.MatchTerms(index.Field{
- Key: eq.Key.ToIndex(seriesID),
+ Key: eq.Key.toIndex(seriesID),
Term: bytes.Join(eq.Expr.Bytes(), nil),
})
}
@@ -490,7 +490,7 @@ func (match *match) Execute(searcher index.GetSearcher, seriesID common.SeriesID
matches[i] = string(v)
}
return s.Match(
- match.Key.ToIndex(seriesID),
+ match.Key.toIndex(seriesID),
matches,
)
}
@@ -524,7 +524,7 @@ func (r *rangeOp) Execute(searcher index.GetSearcher, seriesID common.SeriesID)
if err != nil {
return nil, err
}
- return s.Range(r.Key.ToIndex(seriesID), r.Opts)
+ return s.Range(r.Key.toIndex(seriesID), r.Opts)
}
func (r *rangeOp) MarshalJSON() ([]byte, error) {
@@ -571,7 +571,7 @@ var (
type emptyNode struct{}
-func (an emptyNode) Execute(searcher index.GetSearcher, seriesID common.SeriesID) (posting.List, error) {
+func (an emptyNode) Execute(_ index.GetSearcher, _ common.SeriesID) (posting.List, error) {
return bList, nil
}
@@ -581,7 +581,7 @@ func (an emptyNode) String() string {
type bypassList struct{}
-func (bl bypassList) Contains(id common.ItemID) bool {
+func (bl bypassList) Contains(_ common.ItemID) bool {
// all items should be fetched
return true
}
@@ -606,39 +606,39 @@ func (bl bypassList) Clone() posting.List {
panic("not invoked")
}
-func (bl bypassList) Equal(other posting.List) bool {
+func (bl bypassList) Equal(_ posting.List) bool {
panic("not invoked")
}
-func (bl bypassList) Insert(i common.ItemID) {
+func (bl bypassList) Insert(_ common.ItemID) {
panic("not invoked")
}
-func (bl bypassList) Intersect(other posting.List) error {
+func (bl bypassList) Intersect(_ posting.List) error {
panic("not invoked")
}
-func (bl bypassList) Difference(other posting.List) error {
+func (bl bypassList) Difference(_ posting.List) error {
panic("not invoked")
}
-func (bl bypassList) Union(other posting.List) error {
+func (bl bypassList) Union(_ posting.List) error {
panic("not invoked")
}
-func (bl bypassList) UnionMany(others []posting.List) error {
+func (bl bypassList) UnionMany(_ []posting.List) error {
panic("not invoked")
}
-func (bl bypassList) AddIterator(iter posting.Iterator) error {
+func (bl bypassList) AddIterator(_ posting.Iterator) error {
panic("not invoked")
}
-func (bl bypassList) AddRange(min common.ItemID, max common.ItemID) error {
+func (bl bypassList) AddRange(_ common.ItemID, _ common.ItemID) error {
panic("not invoked")
}
-func (bl bypassList) RemoveRange(min common.ItemID, max common.ItemID) error {
+func (bl bypassList) RemoveRange(_ common.ItemID, _ common.ItemID) error {
panic("not invoked")
}
@@ -654,7 +654,7 @@ func (bl bypassList) Marshall() ([]byte, error) {
panic("not invoked")
}
-func (bl bypassList) Unmarshall(data []byte) error {
+func (bl bypassList) Unmarshall(_ []byte) error {
panic("not invoked")
}
diff --git a/pkg/query/logical/interface.go b/pkg/query/logical/interface.go
index 6be21ac..dc002ca 100644
--- a/pkg/query/logical/interface.go
+++ b/pkg/query/logical/interface.go
@@ -21,32 +21,33 @@ import (
"fmt"
)
+// UnresolvedPlan denotes an logical expression.
+// It could be analyzed to a Plan(executable operation) with the Schema.
type UnresolvedPlan interface {
Analyze(Schema) (Plan, error)
}
+// Plan is the executable operation. It belongs to a execution tree.
type Plan interface {
fmt.Stringer
Children() []Plan
Schema() Schema
}
+// Expr represents a predicate in criteria.
type Expr interface {
fmt.Stringer
DataType() int32
Equal(Expr) bool
}
+// LiteralExpr allows getting raw data represented as bytes.
type LiteralExpr interface {
Expr
Bytes() [][]byte
}
-type ResolvableExpr interface {
- Expr
- Resolve(Schema) error
-}
-
+// ComparableExpr allows comparing Expr and Expr arrays.
type ComparableExpr interface {
LiteralExpr
Compare(LiteralExpr) (int, bool)
diff --git a/pkg/query/logical/iter.go b/pkg/query/logical/iter.go
index 29d00fc..73d1a54 100644
--- a/pkg/query/logical/iter.go
+++ b/pkg/query/logical/iter.go
@@ -20,11 +20,13 @@ package logical
import (
"container/heap"
+ modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
)
var _ ItemIterator = (*itemIter)(nil)
+// ItemIterator allow iterating over a tsdb's series.
type ItemIterator interface {
HasNext() bool
Next() tsdb.Item
@@ -34,7 +36,7 @@ var _ heap.Interface = (*containerHeap)(nil)
// container contains both iter and its current item.
type container struct {
- c Comparator
+ c comparator
item tsdb.Item
iter tsdb.Iterator
}
@@ -58,14 +60,15 @@ func (h *containerHeap) Pop() interface{} {
}
type itemIter struct {
- c Comparator
+ c comparator
h *containerHeap
iters []tsdb.Iterator
}
-func NewItemIter(iters []tsdb.Iterator, c Comparator) ItemIterator {
+// NewItemIter returns a ItemIterator which mergers several tsdb.Iterator by input sorting order.
+func NewItemIter(iters []tsdb.Iterator, sort modelv1.Sort) ItemIterator {
it := &itemIter{
- c: c,
+ c: createComparator(sort),
iters: iters,
h: &containerHeap{},
}
diff --git a/pkg/query/logical/measure/measure_analyzer.go b/pkg/query/logical/measure/measure_analyzer.go
index 312ab87..412362f 100644
--- a/pkg/query/logical/measure/measure_analyzer.go
+++ b/pkg/query/logical/measure/measure_analyzer.go
@@ -27,16 +27,19 @@ import (
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
+// Analyzer analyzes the measure querying expression to the execution plan.
type Analyzer struct {
metadataRepoImpl metadata.Repo
}
+// CreateAnalyzerFromMetaService returns a Analyzer.
func CreateAnalyzerFromMetaService(metaSvc metadata.Service) (*Analyzer, error) {
return &Analyzer{
metaSvc,
}, nil
}
+// BuildSchema returns Schema loaded from the metadata repository.
func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata) (logical.Schema, error) {
group, err := a.metadataRepoImpl.GroupRegistry().GetGroup(ctx, metadata.GetGroup())
if err != nil {
@@ -76,6 +79,7 @@ func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata)
return ms, nil
}
+// Analyze converts logical expressions to executable operation tree represented by Plan.
func (a *Analyzer) Analyze(_ context.Context, criteria *measurev1.QueryRequest, metadata *commonv1.Metadata, s logical.Schema) (logical.Plan, error) {
groupByEntity := false
var groupByTags [][]*logical.Tag
@@ -99,11 +103,11 @@ func (a *Analyzer) Analyze(_ context.Context, criteria *measurev1.QueryRequest,
}
if criteria.GetGroupBy() != nil {
- plan = GroupBy(plan, groupByTags, groupByEntity)
+ plan = newUnresolvedGroupBy(plan, groupByTags, groupByEntity)
}
if criteria.GetAgg() != nil {
- plan = Aggregation(plan,
+ plan = newUnresolvedAggregation(plan,
logical.NewField(criteria.GetAgg().GetFieldName()),
criteria.GetAgg().GetFunction(),
criteria.GetGroupBy() != nil,
@@ -111,7 +115,7 @@ func (a *Analyzer) Analyze(_ context.Context, criteria *measurev1.QueryRequest,
}
if criteria.GetTop() != nil {
- plan = Top(plan, criteria.GetTop())
+ plan = top(plan, criteria.GetTop())
}
// parse limit and offset
@@ -119,7 +123,7 @@ func (a *Analyzer) Analyze(_ context.Context, criteria *measurev1.QueryRequest,
if limitParameter == 0 {
limitParameter = logical.DefaultLimit
}
- plan = Limit(plan, criteria.GetOffset(), limitParameter)
+ plan = limit(plan, criteria.GetOffset(), limitParameter)
return plan.Analyze(s)
}
@@ -165,6 +169,6 @@ func parseFields(criteria *measurev1.QueryRequest, metadata *commonv1.Metadata,
unresolvedOrderBy = logical.NewOrderBy(queryOrder.GetIndexRuleName(), queryOrder.GetSort())
}
- return IndexScan(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
+ return indexScan(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
filter, entities, projTags, projFields, groupByEntity, unresolvedOrderBy), nil
}
diff --git a/pkg/query/logical/measure/measure_plan.go b/pkg/query/logical/measure/measure_plan.go
index 8460711..870e5af 100644
--- a/pkg/query/logical/measure/measure_plan.go
+++ b/pkg/query/logical/measure/measure_plan.go
@@ -66,7 +66,7 @@ func (l *limitPlan) Children() []logical.Plan {
return []logical.Plan{l.Input}
}
-func Limit(input logical.UnresolvedPlan, offset, limit uint32) logical.UnresolvedPlan {
+func limit(input logical.UnresolvedPlan, offset, limit uint32) logical.UnresolvedPlan {
return &limitPlan{
Parent: &logical.Parent{
UnresolvedInput: input,
diff --git a/pkg/query/logical/measure/measure_plan_aggregation.go b/pkg/query/logical/measure/measure_plan_aggregation.go
index 4760210..9c35adc 100644
--- a/pkg/query/logical/measure/measure_plan_aggregation.go
+++ b/pkg/query/logical/measure/measure_plan_aggregation.go
@@ -41,7 +41,7 @@ type unresolvedAggregation struct {
isGroup bool
}
-func Aggregation(input logical.UnresolvedPlan, aggrField *logical.Field, aggrFunc modelv1.AggregationFunction, isGroup bool) logical.UnresolvedPlan {
+func newUnresolvedAggregation(input logical.UnresolvedPlan, aggrField *logical.Field, aggrFunc modelv1.AggregationFunction, isGroup bool) logical.UnresolvedPlan {
return &unresolvedAggregation{
unresolvedInput: input,
aggrFunc: aggrFunc,
@@ -61,7 +61,7 @@ func (gba *unresolvedAggregation) Analyze(measureSchema logical.Schema) (logical
return nil, err
}
if len(aggregationFieldRefs) == 0 {
- return nil, errors.Wrap(logical.ErrFieldNotDefined, "aggregation schema")
+ return nil, errors.Wrap(errFieldNotDefined, "aggregation schema")
}
aggrFunc, err := aggregation.NewInt64Func(gba.aggrFunc)
if err != nil {
diff --git a/pkg/query/logical/measure/measure_plan_groupby.go b/pkg/query/logical/measure/measure_plan_groupby.go
index 26e5a0b..9779e85 100644
--- a/pkg/query/logical/measure/measure_plan_groupby.go
+++ b/pkg/query/logical/measure/measure_plan_groupby.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package measure
import (
@@ -42,7 +43,7 @@ type unresolvedGroup struct {
groupByEntity bool
}
-func GroupBy(input logical.UnresolvedPlan, groupBy [][]*logical.Tag, groupByEntity bool) logical.UnresolvedPlan {
+func newUnresolvedGroupBy(input logical.UnresolvedPlan, groupBy [][]*logical.Tag, groupByEntity bool) logical.UnresolvedPlan {
return &unresolvedGroup{
unresolvedInput: input,
groupBy: groupBy,
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index 96f0fc6..27c1fa6 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -124,7 +124,7 @@ func (i *localIndexScan) Execute(ec executor.MeasureExecutionContext) (executor.
}
}
if len(seriesList) == 0 {
- return executor.EmptyMIterator, nil
+ return dummyIter, nil
}
var builders []logical.SeekerBuilder
if i.Index != nil {
@@ -154,7 +154,7 @@ func (i *localIndexScan) Execute(ec executor.MeasureExecutionContext) (executor.
}
if len(iters) == 0 {
- return executor.EmptyMIterator, nil
+ return dummyIter, nil
}
transformContext := transformContext{
ec: ec,
@@ -164,8 +164,7 @@ func (i *localIndexScan) Execute(ec executor.MeasureExecutionContext) (executor.
if i.groupByEntity {
return newSeriesMIterator(iters, transformContext), nil
}
- c := logical.CreateComparator(i.Sort)
- it := logical.NewItemIter(iters, c)
+ it := logical.NewItemIter(iters, i.Sort)
return newIndexScanIterator(it, transformContext), nil
}
@@ -186,7 +185,7 @@ func (i *localIndexScan) Schema() logical.Schema {
return i.schema.ProjTags(i.projectionTagsRefs...).ProjFields(i.projectionFieldsRefs...)
}
-func IndexScan(startTime, endTime time.Time, metadata *commonv1.Metadata, filter index.Filter, entities []tsdb.Entity,
+func indexScan(startTime, endTime time.Time, metadata *commonv1.Metadata, filter index.Filter, entities []tsdb.Entity,
projectionTags [][]*logical.Tag, projectionFields []*logical.Field, groupByEntity bool, unresolvedOrderBy *logical.UnresolvedOrderBy,
) logical.UnresolvedPlan {
return &unresolvedIndexScan{
@@ -318,3 +317,19 @@ func transform(item tsdb.Item, ism transformContext) (*measurev1.DataPoint, erro
Timestamp: timestamppb.New(time.Unix(0, int64(item.Time()))),
}, nil
}
+
+var dummyIter = dummyMIterator{}
+
+type dummyMIterator struct{}
+
+func (ei dummyMIterator) Next() bool {
+ return false
+}
+
+func (ei dummyMIterator) Current() []*measurev1.DataPoint {
+ return nil
+}
+
+func (ei dummyMIterator) Close() error {
+ return nil
+}
diff --git a/pkg/query/logical/measure/measure_plan_top.go b/pkg/query/logical/measure/measure_plan_top.go
index c774afb..0661c4c 100644
--- a/pkg/query/logical/measure/measure_plan_top.go
+++ b/pkg/query/logical/measure/measure_plan_top.go
@@ -38,7 +38,7 @@ type unresolvedTop struct {
top *measurev1.QueryRequest_Top
}
-func Top(input logical.UnresolvedPlan, top *measurev1.QueryRequest_Top) logical.UnresolvedPlan {
+func top(input logical.UnresolvedPlan, top *measurev1.QueryRequest_Top) logical.UnresolvedPlan {
return &unresolvedTop{
unresolvedInput: input,
top: top,
@@ -55,13 +55,13 @@ func (gba *unresolvedTop) Analyze(measureSchema logical.Schema) (logical.Plan, e
return nil, err
}
if len(fieldRefs) == 0 {
- return nil, errors.Wrap(logical.ErrFieldNotDefined, "top schema")
+ return nil, errors.Wrap(errFieldNotDefined, "top schema")
}
reverted := false
if gba.top.FieldValueSort == modelv1.Sort_SORT_ASC {
reverted = true
}
- return &top{
+ return &topOp{
Parent: &logical.Parent{
UnresolvedInput: gba.unresolvedInput,
Input: prevPlan,
@@ -71,25 +71,25 @@ func (gba *unresolvedTop) Analyze(measureSchema logical.Schema) (logical.Plan, e
}, nil
}
-type top struct {
+type topOp struct {
*logical.Parent
topNStream *TopQueue
fieldRef *logical.FieldRef
}
-func (g *top) String() string {
+func (g *topOp) String() string {
return fmt.Sprintf("%s top %s", g.Input, g.topNStream.String())
}
-func (g *top) Children() []logical.Plan {
+func (g *topOp) Children() []logical.Plan {
return []logical.Plan{g.Input}
}
-func (g *top) Schema() logical.Schema {
+func (g *topOp) Schema() logical.Schema {
return g.Input.Schema()
}
-func (g *top) Execute(ec executor.MeasureExecutionContext) (executor.MIterator, error) {
+func (g *topOp) Execute(ec executor.MeasureExecutionContext) (executor.MIterator, error) {
iter, err := g.Parent.Input.(executor.MeasureExecutable).Execute(ec)
if err != nil {
return nil, err
diff --git a/pkg/query/logical/measure/measure_top.go b/pkg/query/logical/measure/measure_top.go
index cc6487c..39bec10 100644
--- a/pkg/query/logical/measure/measure_top.go
+++ b/pkg/query/logical/measure/measure_top.go
@@ -25,11 +25,13 @@ import (
measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
)
+// TopElement seals a sortable value and its data point which this value belongs to.
type TopElement struct {
dp *measurev1.DataPoint
value int64
}
+// NewTopElement returns a TopElement.
func NewTopElement(dp *measurev1.DataPoint, value int64) TopElement {
return TopElement{
dp: dp,
@@ -37,6 +39,7 @@ func NewTopElement(dp *measurev1.DataPoint, value int64) TopElement {
}
}
+// Val returns the sortable value.
func (e TopElement) Val() int64 {
return e.value
}
@@ -91,11 +94,13 @@ func (h *topHeap) Pop() interface{} {
return e
}
+// TopQueue is a sortable queue only keeps top-n members when pushed new elements.
type TopQueue struct {
th topHeap
n int
}
+// NewTopQueue returns a new TopQueue.
func NewTopQueue(n int, reverted bool) *TopQueue {
return &TopQueue{
n: n,
@@ -106,6 +111,9 @@ func NewTopQueue(n int, reverted bool) *TopQueue {
}
}
+// Insert pushes a new element to the queue.
+// It returns true if the element are accepted by the queue,
+// returns false if it's evicted.
func (s *TopQueue) Insert(element TopElement) bool {
if len(s.th.elements) < s.n {
heap.Push(&s.th, element)
@@ -127,10 +135,12 @@ func (s *TopQueue) Insert(element TopElement) bool {
return true
}
+// Purge resets the queue.
func (s *TopQueue) Purge() {
s.th.elements = s.th.elements[:0]
}
+// Elements returns all elements accepted by the queue.
func (s *TopQueue) Elements() []TopElement {
l := &topSortedList{
elements: append([]TopElement(nil), s.th.elements...),
@@ -140,6 +150,7 @@ func (s *TopQueue) Elements() []TopElement {
return l.elements
}
+// Strings shows the string represent.
func (s TopQueue) String() string {
if s.th.reverted {
return fmt.Sprintf("bottom(%d)", s.n)
@@ -147,6 +158,7 @@ func (s TopQueue) String() string {
return fmt.Sprintf("top(%d)", s.n)
}
+// Equal reports whether s and other have the queue's max acceptable number and sorting order.
func (s *TopQueue) Equal(other *TopQueue) bool {
return s.th.reverted == other.th.reverted && s.n == other.n
}
diff --git a/pkg/query/logical/measure/schema.go b/pkg/query/logical/measure/schema.go
index a3e1ed6..7161a09 100644
--- a/pkg/query/logical/measure/schema.go
+++ b/pkg/query/logical/measure/schema.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package measure implements execution operations for querying measure data.
package measure
import (
@@ -26,6 +27,8 @@ import (
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
+var errFieldNotDefined = errors.New("field is not defined")
+
type schema struct {
measure *databasev1.Measure
fieldMap map[string]*logical.FieldSpec
@@ -58,7 +61,7 @@ func (m *schema) CreateFieldRef(fields ...*logical.Field) ([]*logical.FieldRef,
if fs, ok := m.fieldMap[field.Name]; ok {
fieldRefs[idx] = &logical.FieldRef{Field: field, Spec: fs}
} else {
- return nil, errors.Wrap(logical.ErrFieldNotDefined, field.Name)
+ return nil, errors.Wrap(errFieldNotDefined, field.Name)
}
}
return fieldRefs, nil
diff --git a/pkg/query/logical/plan.go b/pkg/query/logical/plan.go
index f6313c6..7787659 100644
--- a/pkg/query/logical/plan.go
+++ b/pkg/query/logical/plan.go
@@ -25,21 +25,22 @@ import (
)
var (
- _ Plan = (*Limit)(nil)
- _ UnresolvedPlan = (*Limit)(nil)
+ _ Plan = (*limit)(nil)
+ _ UnresolvedPlan = (*limit)(nil)
)
+// Parent refers to a parent node in the execution tree(plan).
type Parent struct {
UnresolvedInput UnresolvedPlan
Input Plan
}
-type Limit struct {
+type limit struct {
*Parent
LimitNum uint32
}
-func (l *Limit) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element, error) {
+func (l *limit) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element, error) {
entities, err := l.Parent.Input.(executor.StreamExecutable).Execute(ec)
if err != nil {
return nil, err
@@ -52,7 +53,7 @@ func (l *Limit) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element
return entities, nil
}
-func (l *Limit) Analyze(s Schema) (Plan, error) {
+func (l *limit) Analyze(s Schema) (Plan, error) {
var err error
l.Input, err = l.UnresolvedInput.Analyze(s)
if err != nil {
@@ -61,20 +62,21 @@ func (l *Limit) Analyze(s Schema) (Plan, error) {
return l, nil
}
-func (l *Limit) Schema() Schema {
+func (l *limit) Schema() Schema {
return l.Input.Schema()
}
-func (l *Limit) String() string {
+func (l *limit) String() string {
return fmt.Sprintf("%s Limit: %d", l.Input.String(), l.LimitNum)
}
-func (l *Limit) Children() []Plan {
+func (l *limit) Children() []Plan {
return []Plan{l.Input}
}
+// NewLimit return a limitation expression.
func NewLimit(input UnresolvedPlan, num uint32) UnresolvedPlan {
- return &Limit{
+ return &limit{
Parent: &Parent{
UnresolvedInput: input,
},
@@ -83,16 +85,16 @@ func NewLimit(input UnresolvedPlan, num uint32) UnresolvedPlan {
}
var (
- _ Plan = (*Offset)(nil)
- _ UnresolvedPlan = (*Offset)(nil)
+ _ Plan = (*offset)(nil)
+ _ UnresolvedPlan = (*offset)(nil)
)
-type Offset struct {
+type offset struct {
*Parent
offsetNum uint32
}
-func (l *Offset) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element, error) {
+func (l *offset) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Element, error) {
elements, err := l.Parent.Input.(executor.StreamExecutable).Execute(ec)
if err != nil {
return nil, err
@@ -105,7 +107,7 @@ func (l *Offset) Execute(ec executor.StreamExecutionContext) ([]*streamv1.Elemen
return []*streamv1.Element{}, nil
}
-func (l *Offset) Analyze(s Schema) (Plan, error) {
+func (l *offset) Analyze(s Schema) (Plan, error) {
var err error
l.Input, err = l.UnresolvedInput.Analyze(s)
if err != nil {
@@ -114,20 +116,21 @@ func (l *Offset) Analyze(s Schema) (Plan, error) {
return l, nil
}
-func (l *Offset) Schema() Schema {
+func (l *offset) Schema() Schema {
return l.Input.Schema()
}
-func (l *Offset) String() string {
+func (l *offset) String() string {
return fmt.Sprintf("%s Offset: %d", l.Input.String(), l.offsetNum)
}
-func (l *Offset) Children() []Plan {
+func (l *offset) Children() []Plan {
return []Plan{l.Input}
}
+// NewOffset returns a offset expression.
func NewOffset(input UnresolvedPlan, num uint32) UnresolvedPlan {
- return &Offset{
+ return &offset{
Parent: &Parent{
UnresolvedInput: input,
},
diff --git a/pkg/query/logical/plan_orderby.go b/pkg/query/logical/plan_orderby.go
index dfe38a2..28071f0 100644
--- a/pkg/query/logical/plan_orderby.go
+++ b/pkg/query/logical/plan_orderby.go
@@ -18,23 +18,21 @@
package logical
import (
- "bytes"
"fmt"
- "sort"
"github.com/pkg/errors"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
- "github.com/apache/skywalking-banyandb/pkg/convert"
)
+// UnresolvedOrderBy is unanalyzed sorting expr.
type UnresolvedOrderBy struct {
targetIndexRuleName string
sort modelv1.Sort
}
+// Analyze the UnresolvedOrderBy to get OrderBy result.
func (u *UnresolvedOrderBy) Analyze(s Schema) (*OrderBy, error) {
if u == nil {
// return a default orderBy sub-plan
@@ -51,12 +49,12 @@ func (u *UnresolvedOrderBy) Analyze(s Schema) (*OrderBy, error) {
defined, indexRule := s.IndexRuleDefined(u.targetIndexRuleName)
if !defined {
- return nil, errors.Wrap(ErrIndexNotDefined, u.targetIndexRuleName)
+ return nil, errors.Wrap(errIndexNotDefined, u.targetIndexRuleName)
}
projFieldSpecs, err := s.CreateTagRef(NewTags("", indexRule.GetTags()...))
if err != nil {
- return nil, ErrTagNotDefined
+ return nil, errTagNotDefined
}
return &OrderBy{
@@ -66,12 +64,14 @@ func (u *UnresolvedOrderBy) Analyze(s Schema) (*OrderBy, error) {
}, nil
}
+// OrderBy is the sorting operator.
type OrderBy struct {
Index *databasev1.IndexRule
fieldRefs []*TagRef
Sort modelv1.Sort
}
+// Equal reports whether o and other has the same sorting order and name.
func (o *OrderBy) Equal(other interface{}) bool {
if otherOrderBy, ok := other.(*OrderBy); ok {
if o == nil && otherOrderBy == nil {
@@ -87,80 +87,15 @@ func (o *OrderBy) Equal(other interface{}) bool {
return false
}
+// Strings shows the string represent.
func (o *OrderBy) String() string {
return fmt.Sprintf("OrderBy: %v, sort=%s", o.Index.GetTags(), o.Sort.String())
}
+// NewOrderBy return a sorting expr.
func NewOrderBy(indexRuleName string, sort modelv1.Sort) *UnresolvedOrderBy {
return &UnresolvedOrderBy{
sort: sort,
targetIndexRuleName: indexRuleName,
}
}
-
-func getRawTagValue(typedPair *modelv1.Tag) ([]byte, error) {
- switch v := typedPair.GetValue().Value.(type) {
- case *modelv1.TagValue_Str:
- return []byte(v.Str.GetValue()), nil
- case *modelv1.TagValue_Int:
- return convert.Int64ToBytes(v.Int.GetValue()), nil
- default:
- return nil, errors.New("unsupported data types")
- }
-}
-
-// SortedByIndex is used to test whether the given entities are sorted by the sortDirection
-// The given entities MUST satisfy both the positive check and the negative check for the reversed direction.
-func SortedByIndex(elements []*streamv1.Element, tagFamilyIdx, tagIdx int, sortDirection modelv1.Sort) bool {
- if modelv1.Sort_SORT_UNSPECIFIED == sortDirection {
- sortDirection = modelv1.Sort_SORT_ASC
- }
- if len(elements) == 1 {
- return true
- }
- return sort.SliceIsSorted(elements, sortByIndex(elements, tagFamilyIdx, tagIdx, sortDirection)) &&
- !sort.SliceIsSorted(elements, sortByIndex(elements, tagFamilyIdx, tagIdx, reverseSortDirection(sortDirection)))
-}
-
-func SortedByTimestamp(elements []*streamv1.Element, sortDirection modelv1.Sort) bool {
- if modelv1.Sort_SORT_UNSPECIFIED == sortDirection {
- sortDirection = modelv1.Sort_SORT_ASC
- }
- if len(elements) == 1 {
- return true
- }
- return sort.SliceIsSorted(elements, sortByTimestamp(elements, sortDirection)) &&
- !sort.SliceIsSorted(elements, sortByTimestamp(elements, reverseSortDirection(sortDirection)))
-}
-
-func reverseSortDirection(sort modelv1.Sort) modelv1.Sort {
- if sort == modelv1.Sort_SORT_DESC {
- return modelv1.Sort_SORT_ASC
- }
- return modelv1.Sort_SORT_DESC
-}
-
-func sortByIndex(entities []*streamv1.Element, tagFamilyIdx, tagIdx int, sortDirection modelv1.Sort) func(i, j int) bool {
- return func(i, j int) bool {
- iPair := entities[i].GetTagFamilies()[tagFamilyIdx].GetTags()[tagIdx]
- jPair := entities[j].GetTagFamilies()[tagFamilyIdx].GetTags()[tagIdx]
- lField, _ := getRawTagValue(iPair)
- rField, _ := getRawTagValue(jPair)
- comp := bytes.Compare(lField, rField)
- if sortDirection == modelv1.Sort_SORT_ASC {
- return comp == -1
- }
- return comp == 1
- }
-}
-
-func sortByTimestamp(entities []*streamv1.Element, sortDirection modelv1.Sort) func(i, j int) bool {
- return func(i, j int) bool {
- iPair := entities[i].GetTimestamp().AsTime().UnixNano()
- jPair := entities[j].GetTimestamp().AsTime().UnixNano()
- if sortDirection == modelv1.Sort_SORT_DESC {
- return iPair > jPair
- }
- return iPair < jPair
- }
-}
diff --git a/pkg/query/logical/schema.go b/pkg/query/logical/schema.go
index dac7660..fedac76 100644
--- a/pkg/query/logical/schema.go
+++ b/pkg/query/logical/schema.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/skywalking-banyandb/banyand/tsdb"
)
+// Schema allows retrieving schemas in a convenient way.
type Schema interface {
Scope() tsdb.Entry
EntityList() []string
@@ -38,17 +39,21 @@ type Schema interface {
ShardNumber() uint32
}
+// TagSpec wraps offsets to access a tag in the raw data swiftly.
type TagSpec struct {
Spec *databasev1.TagSpec
TagFamilyIdx int
TagIdx int
}
+// Equal compares fs and other have the same fields.
func (fs *TagSpec) Equal(other *TagSpec) bool {
return fs.TagFamilyIdx == other.TagFamilyIdx && fs.TagIdx == other.TagIdx &&
fs.Spec.GetType() == other.Spec.GetType() && fs.Spec.GetName() == other.Spec.GetName()
}
+// CommonSchema represents a sharable fields between independent schemas.
+// It provides common access methods at the same time.
type CommonSchema struct {
Group *commonv1.Group
IndexRules []*databasev1.IndexRule
@@ -56,6 +61,7 @@ type CommonSchema struct {
EntityList []string
}
+// ProjTags inits a dictionary for getting TagSpec by tag's name.
func (cs *CommonSchema) ProjTags(refs ...[]*TagRef) *CommonSchema {
if len(refs) == 0 {
return nil
@@ -67,7 +73,7 @@ func (cs *CommonSchema) ProjTags(refs ...[]*TagRef) *CommonSchema {
}
for projFamilyIdx, refInFamily := range refs {
for projIdx, ref := range refInFamily {
- newCommonSchema.TagMap[ref.Tag.GetTagName()] = &TagSpec{
+ newCommonSchema.TagMap[ref.Tag.getTagName()] = &TagSpec{
TagFamilyIdx: projFamilyIdx,
TagIdx: projIdx,
Spec: ref.Spec.Spec,
@@ -77,7 +83,7 @@ func (cs *CommonSchema) ProjTags(refs ...[]*TagRef) *CommonSchema {
return newCommonSchema
}
-// registerTag registers the tag spec with given tagFamilyName, tagName and indexes.
+// RegisterTag registers the tag spec with given tagFamilyName, tagName and indexes.
func (cs *CommonSchema) RegisterTag(tagFamilyIdx, tagIdx int, spec *databasev1.TagSpec) {
cs.TagMap[spec.GetName()] = &TagSpec{
TagIdx: tagIdx,
@@ -86,6 +92,7 @@ func (cs *CommonSchema) RegisterTag(tagFamilyIdx, tagIdx int, spec *databasev1.T
}
}
+// ShardNumber returns the number of shards defined by the schema.
func (cs *CommonSchema) ShardNumber() uint32 {
return cs.Group.ResourceOpts.ShardNum
}
@@ -102,6 +109,7 @@ func (cs *CommonSchema) IndexDefined(tagName string) (bool, *databasev1.IndexRul
return false, nil
}
+// IndexRuleDefined return the IndexRule by its name.
func (cs *CommonSchema) IndexRuleDefined(indexRuleName string) (bool, *databasev1.IndexRule) {
for _, idxRule := range cs.IndexRules {
if idxRule.GetMetadata().GetName() == indexRuleName {
@@ -119,10 +127,10 @@ func (cs *CommonSchema) CreateRef(tags ...[]*Tag) ([][]*TagRef, error) {
for i, tagInFamily := range tags {
var tagRefsInFamily []*TagRef
for _, tag := range tagInFamily {
- if ts, ok := cs.TagMap[tag.GetTagName()]; ok {
+ if ts, ok := cs.TagMap[tag.getTagName()]; ok {
tagRefsInFamily = append(tagRefsInFamily, &TagRef{tag, ts})
} else {
- return nil, errors.Wrap(ErrTagNotDefined, tag.GetCompoundName())
+ return nil, errors.Wrap(errTagNotDefined, tag.GetCompoundName())
}
}
tagRefs[i] = tagRefsInFamily
diff --git a/pkg/query/logical/stream/schema.go b/pkg/query/logical/stream/schema.go
index b7004d7..3961fb3 100644
--- a/pkg/query/logical/stream/schema.go
+++ b/pkg/query/logical/stream/schema.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package stream implements execution operations for querying stream data.
package stream
import (
@@ -32,7 +33,7 @@ type schema struct {
common *logical.CommonSchema
}
-func (s *schema) CreateFieldRef(fields ...*logical.Field) ([]*logical.FieldRef, error) {
+func (s *schema) CreateFieldRef(_ ...*logical.Field) ([]*logical.FieldRef, error) {
panic("no field for stream")
}
diff --git a/pkg/query/logical/stream/stream_analyzer.go b/pkg/query/logical/stream/stream_analyzer.go
index 2beacc1..2087bf4 100644
--- a/pkg/query/logical/stream/stream_analyzer.go
+++ b/pkg/query/logical/stream/stream_analyzer.go
@@ -26,16 +26,19 @@ import (
"github.com/apache/skywalking-banyandb/pkg/query/logical"
)
+// Analyzer analyzes the stream querying expression to the execution plan.
type Analyzer struct {
metadataRepoImpl metadata.Repo
}
+// CreateAnalyzerFromMetaService returns a Analyzer.
func CreateAnalyzerFromMetaService(metaSvc metadata.Service) (*Analyzer, error) {
return &Analyzer{
metaSvc,
}, nil
}
+// BuildSchema returns Schema loaded from the metadata repository.
func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata) (logical.Schema, error) {
group, err := a.metadataRepoImpl.GroupRegistry().GetGroup(ctx, metadata.GetGroup())
if err != nil {
@@ -71,6 +74,7 @@ func (a *Analyzer) BuildSchema(ctx context.Context, metadata *commonv1.Metadata)
return s, nil
}
+// Analyze converts logical expressions to executable operation tree represented by Plan.
func (a *Analyzer) Analyze(_ context.Context, criteria *streamv1.QueryRequest, metadata *commonv1.Metadata, s logical.Schema) (logical.Plan, error) {
// parse fields
plan := parseTags(criteria, metadata)
@@ -115,6 +119,6 @@ func parseTags(criteria *streamv1.QueryRequest, metadata *commonv1.Metadata) log
projTags[i] = projTagInFamily
}
- return TagFilter(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
+ return tagFilter(timeRange.GetBegin().AsTime(), timeRange.GetEnd().AsTime(), metadata,
criteria.Criteria, nil, projTags...)
}
diff --git a/pkg/query/logical/stream/stream_plan_indexscan_local.go b/pkg/query/logical/stream/stream_plan_indexscan_local.go
index 80bd20f..00efdb0 100644
--- a/pkg/query/logical/stream/stream_plan_indexscan_local.go
+++ b/pkg/query/logical/stream/stream_plan_indexscan_local.go
@@ -103,8 +103,7 @@ func (i *localIndexScan) Execute(ec executor.StreamExecutionContext) ([]*streamv
return elems, nil
}
- c := logical.CreateComparator(i.Sort)
- it := logical.NewItemIter(iters, c)
+ it := logical.NewItemIter(iters, i.Sort)
for it.HasNext() {
nextItem := it.Next()
tagFamilies, innerErr := logical.ProjectItem(ec, nextItem, i.projectionTagRefs)
diff --git a/pkg/query/logical/stream/stream_plan_tag_filter.go b/pkg/query/logical/stream/stream_plan_tag_filter.go
index 332b999..5e8a461 100644
--- a/pkg/query/logical/stream/stream_plan_tag_filter.go
+++ b/pkg/query/logical/stream/stream_plan_tag_filter.go
@@ -34,7 +34,10 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-var _ logical.UnresolvedPlan = (*unresolvedTagFilter)(nil)
+var (
+ _ logical.UnresolvedPlan = (*unresolvedTagFilter)(nil)
+ errMultipleGlobalIndexes = errors.New("multiple global indexes are not supported")
+)
type unresolvedTagFilter struct {
startTime time.Time
@@ -58,8 +61,8 @@ func (uis *unresolvedTagFilter) Analyze(s logical.Schema) (logical.Plan, error)
var err error
ctx.filter, ctx.entities, err = logical.BuildLocalFilter(uis.criteria, s, entityDict, entity)
if err != nil {
- ge := &logical.GlobalIndexError{}
- if errors.As(err, ge) {
+ var ge logical.GlobalIndexError
+ if errors.As(err, &ge) {
ctx.globalConditions = append(ctx.globalConditions, ge.IndexRule, ge.Expr)
} else {
return nil, err
@@ -82,8 +85,8 @@ func (uis *unresolvedTagFilter) Analyze(s logical.Schema) (logical.Plan, error)
if errFilter != nil {
return nil, errFilter
}
- if tagFilter != logical.BypassFilter {
- plan = NewTagFilter(s, plan, tagFilter)
+ if tagFilter != logical.DummyFilter {
+ plan = newTagFilter(s, plan, tagFilter)
}
}
return plan, err
@@ -92,7 +95,7 @@ func (uis *unresolvedTagFilter) Analyze(s logical.Schema) (logical.Plan, error)
func (uis *unresolvedTagFilter) selectIndexScanner(ctx *analyzeContext) (logical.Plan, error) {
if len(ctx.globalConditions) > 0 {
if len(ctx.globalConditions) > 2 {
- return nil, logical.ErrMultipleGlobalIndexes
+ return nil, errMultipleGlobalIndexes
}
return &globalIndexScan{
schema: ctx.s,
@@ -121,7 +124,7 @@ func (uis *unresolvedTagFilter) selectIndexScanner(ctx *analyzeContext) (logical
}, nil
}
-func TagFilter(startTime, endTime time.Time, metadata *commonv1.Metadata, criteria *modelv1.Criteria,
+func tagFilter(startTime, endTime time.Time, metadata *commonv1.Metadata, criteria *modelv1.Criteria,
orderBy *logical.UnresolvedOrderBy, projection ...[]*logical.Tag,
) logical.UnresolvedPlan {
return &unresolvedTagFilter{
@@ -160,7 +163,7 @@ type tagFilterPlan struct {
tagFilter logical.TagFilter
}
-func NewTagFilter(s logical.Schema, parent logical.Plan, tagFilter logical.TagFilter) logical.Plan {
+func newTagFilter(s logical.Schema, parent logical.Plan, tagFilter logical.TagFilter) logical.Plan {
return &tagFilterPlan{
s: s,
parent: parent,
diff --git a/pkg/query/logical/tag_filter.go b/pkg/query/logical/tag_filter.go
index a5e59f2..15a9548 100644
--- a/pkg/query/logical/tag_filter.go
+++ b/pkg/query/logical/tag_filter.go
@@ -27,16 +27,18 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
-var ErrUnsupportedLogicalOperation = errors.New("unsupported logical operation")
+var errUnsupportedLogicalOperation = errors.New("unsupported logical operation")
+// TagFilter allows matching a tag based on a predicate.
type TagFilter interface {
fmt.Stringer
Match(tagFamilies []*modelv1.TagFamily) (bool, error)
}
+// BuildTagFilter returns a TagFilter if predicates doesn't match any indices.
func BuildTagFilter(criteria *modelv1.Criteria, entityDict map[string]int, schema Schema, hasGlobalIndex bool) (TagFilter, error) {
if criteria == nil {
- return BypassFilter, nil
+ return DummyFilter, nil
}
switch criteria.GetExp().(type) {
case *modelv1.Criteria_Condition:
@@ -46,10 +48,10 @@ func BuildTagFilter(criteria *modelv1.Criteria, entityDict map[string]int, schem
return nil, err
}
if ok, _ := schema.IndexDefined(cond.Name); ok {
- return BypassFilter, nil
+ return DummyFilter, nil
}
if _, ok := entityDict[cond.Name]; ok {
- return BypassFilter, nil
+ return DummyFilter, nil
}
return parseFilter(cond, expr)
case *modelv1.Criteria_Le:
@@ -62,8 +64,8 @@ func BuildTagFilter(criteria *modelv1.Criteria, entityDict map[string]int, schem
if err != nil {
return nil, err
}
- if left == BypassFilter && right == BypassFilter {
- return BypassFilter, nil
+ if left == DummyFilter && right == DummyFilter {
+ return DummyFilter, nil
}
switch le.Op {
case modelv1.LogicalExpression_LOGICAL_OP_AND:
@@ -72,33 +74,33 @@ func BuildTagFilter(criteria *modelv1.Criteria, entityDict map[string]int, schem
return and, nil
case modelv1.LogicalExpression_LOGICAL_OP_OR:
if hasGlobalIndex {
- return nil, errors.WithMessage(ErrUnsupportedLogicalOperation, "global index doesn't support OR")
+ return nil, errors.WithMessage(errUnsupportedLogicalOperation, "global index doesn't support OR")
}
or := newOrLogicalNode(2)
or.append(left).append(right)
return or, nil
}
}
- return nil, ErrInvalidCriteriaType
+ return nil, errInvalidCriteriaType
}
func parseFilter(cond *modelv1.Condition, expr ComparableExpr) (TagFilter, error) {
switch cond.Op {
case modelv1.Condition_BINARY_OP_GT:
- return newRangeTag(cond.Name, RangeOpts{
+ return newRangeTag(cond.Name, rangeOpts{
Lower: expr,
}), nil
case modelv1.Condition_BINARY_OP_GE:
- return newRangeTag(cond.Name, RangeOpts{
+ return newRangeTag(cond.Name, rangeOpts{
IncludesLower: true,
Lower: expr,
}), nil
case modelv1.Condition_BINARY_OP_LT:
- return newRangeTag(cond.Name, RangeOpts{
+ return newRangeTag(cond.Name, rangeOpts{
Upper: expr,
}), nil
case modelv1.Condition_BINARY_OP_LE:
- return newRangeTag(cond.Name, RangeOpts{
+ return newRangeTag(cond.Name, rangeOpts{
IncludesUpper: true,
Upper: expr,
}), nil
@@ -115,7 +117,7 @@ func parseFilter(cond *modelv1.Condition, expr ComparableExpr) (TagFilter, error
case modelv1.Condition_BINARY_OP_NOT_IN:
panic("unimplemented")
default:
- return nil, errors.WithMessagef(ErrUnsupportedConditionOp, "tag filter parses %v", cond)
+ return nil, errors.WithMessagef(errUnsupportedConditionOp, "tag filter parses %v", cond)
}
}
@@ -140,16 +142,17 @@ func parseExpr(value *modelv1.TagValue) (ComparableExpr, error) {
case *modelv1.TagValue_Null:
return nullLiteralExpr, nil
}
- return nil, errors.WithMessagef(ErrUnsupportedConditionValue, "tag filter parses %v", value)
+ return nil, errors.WithMessagef(errUnsupportedConditionValue, "tag filter parses %v", value)
}
-var BypassFilter = new(emptyFilter)
+// DummyFilter matches any predicate.
+var DummyFilter = new(dummyTagFilter)
-type emptyFilter struct{}
+type dummyTagFilter struct{}
-func (emptyFilter) Match(_ []*modelv1.TagFamily) (bool, error) { return true, nil }
+func (dummyTagFilter) Match(_ []*modelv1.TagFamily) (bool, error) { return true, nil }
-func (emptyFilter) String() string { return "bypass" }
+func (dummyTagFilter) String() string { return "dummy" }
type logicalNodeOP interface {
TagFilter
@@ -161,7 +164,7 @@ type logicalNode struct {
}
func (n *logicalNode) append(sub TagFilter) *logicalNode {
- if sub == BypassFilter {
+ if sub == DummyFilter {
return n
}
n.SubNodes = append(n.SubNodes, sub)
@@ -328,7 +331,7 @@ func (eq *eqTag) String() string {
return jsonToString(eq)
}
-type RangeOpts struct {
+type rangeOpts struct {
Upper ComparableExpr
Lower ComparableExpr
IncludesUpper bool
@@ -337,10 +340,10 @@ type RangeOpts struct {
type rangeTag struct {
*tagLeaf
- Opts RangeOpts
+ Opts rangeOpts
}
-func newRangeTag(tagName string, opts RangeOpts) *rangeTag {
+func newRangeTag(tagName string, opts rangeOpts) *rangeTag {
return &rangeTag{
tagLeaf: &tagLeaf{
Name: tagName,
@@ -426,7 +429,7 @@ func tagExpr(tagFamilies []*modelv1.TagFamily, tagName string) (ComparableExpr,
}
}
}
- return nil, ErrTagNotDefined
+ return nil, errTagNotDefined
}
type havingTag struct {
diff --git a/pkg/run/run.go b/pkg/run/run.go
index 5547b6b..c45e59a 100644
--- a/pkg/run/run.go
+++ b/pkg/run/run.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package run implements a lifecycle framework to control modules.
package run
import (
@@ -94,6 +95,7 @@ func (p preRunner) PreRun() error {
return p.fn()
}
+// StopNotify sends the stopped event to the running system.
type StopNotify <-chan struct{}
// Service interface should be implemented by Group Unit objects that need
@@ -130,6 +132,7 @@ type Group struct {
configured bool
}
+// NewGroup return a Group with input name.
func NewGroup(name string) Group {
return Group{
name: name,
@@ -173,6 +176,7 @@ func (g *Group) Register(units ...Unit) []bool {
return hasRegistered
}
+// RegisterFlags returns FlagSet contains Flags in all modules.
func (g *Group) RegisterFlags() *FlagSet {
// run configuration stage
g.f = NewFlagSet(g.name)
@@ -386,6 +390,7 @@ func (g Group) ListUnits() string {
return fmt.Sprintf("Group: %s [%s]%s", g.name, t, s)
}
+// WaitTillReady blocks the goroutine till all modules are ready.
func (g *Group) WaitTillReady() {
<-g.readyCh
}
diff --git a/pkg/run/test.go b/pkg/run/test.go
index 7151c72..1524e73 100644
--- a/pkg/run/test.go
+++ b/pkg/run/test.go
@@ -22,25 +22,27 @@ import (
"sync"
)
-var _ Service = (*Tester)(nil)
+var _ Service = (*tester)(nil)
-type Tester struct {
+type tester struct {
startedNotifier chan struct{}
stopCh chan struct{}
ID string
once sync.Once
}
-func NewTester(id string) *Tester {
- return &Tester{
+// NewTester return a tester module to stop the runtime programmatically.
+func NewTester(id string) (Unit, func()) {
+ t := &tester{
ID: id,
startedNotifier: make(chan struct{}),
stopCh: make(chan struct{}),
once: sync.Once{},
}
+ return t, t.GracefulStop
}
-func (t *Tester) WaitUntilStarted() error {
+func (t *tester) WaitUntilStarted() error {
select {
case err := <-t.stopCh:
return fmt.Errorf("stopped: %v", err)
@@ -49,16 +51,16 @@ func (t *Tester) WaitUntilStarted() error {
}
}
-func (t *Tester) Name() string {
+func (t *tester) Name() string {
return t.ID
}
-func (t *Tester) Serve() StopNotify {
+func (t *tester) Serve() StopNotify {
close(t.startedNotifier)
return t.stopCh
}
-func (t *Tester) GracefulStop() {
+func (t *tester) GracefulStop() {
t.once.Do(func() {
close(t.stopCh)
})
diff --git a/pkg/schema/metadata.go b/pkg/schema/metadata.go
index daa2859..4388e45 100644
--- a/pkg/schema/metadata.go
+++ b/pkg/schema/metadata.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package schema implements a framework to sync schema info from the metadata repository.
package schema
import (
@@ -40,36 +41,45 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
+// EventType defines actions of events.
type EventType uint8
+// EventType support Add/Update and Delete.
+// All events are idempotent.
const (
EventAddOrUpdate EventType = iota
EventDelete
)
+// EventKind defines category of events.
type EventKind uint8
+// This framework groups events to a hierarchy. A group is the root node.
const (
EventKindGroup EventKind = iota
EventKindResource
)
+// Group is the root node, allowing get resources from its sub nodes.
type Group interface {
GetSchema() *commonv1.Group
StoreResource(resourceSchema ResourceSchema) (Resource, error)
LoadResource(name string) (Resource, bool)
}
+// MetadataEvent is the syncing message between metadata repo and this framework.
type MetadataEvent struct {
Metadata *commonv1.Metadata
Typ EventType
Kind EventKind
}
+// ResourceSchema allows get the metadata.
type ResourceSchema interface {
GetMetadata() *commonv1.Metadata
}
+// ResourceSpec wraps required fields to open a resource.
type ResourceSpec struct {
Schema ResourceSchema
// IndexRules are index rules bound to the Schema
@@ -78,6 +88,7 @@ type ResourceSpec struct {
Aggregations []*databasev1.TopNAggregation
}
+// Resource allows access metadata from a local cache.
type Resource interface {
GetIndexRules() []*databasev1.IndexRule
MaxObservedModRevision() int64
@@ -86,12 +97,14 @@ type Resource interface {
io.Closer
}
+// ResourceSupplier allows open a resource and its embedded tsdb.
type ResourceSupplier interface {
OpenResource(shardNum uint32, db tsdb.Supplier, spec ResourceSpec) (Resource, error)
ResourceSchema(metdata *commonv1.Metadata) (ResourceSchema, error)
OpenDB(groupSchema *commonv1.Group) (tsdb.Database, error)
}
+// Repository is the collection of several hierarchies groups by a "Group".
type Repository interface {
Watcher()
SendMetadataEvent(MetadataEvent)
@@ -117,6 +130,7 @@ type schemaRepo struct {
sync.RWMutex
}
+// NewRepository return a new Repository.
func NewRepository(
metadata metadata.Repo,
repo discovery.ServiceRepo,
diff --git a/pkg/signal/handler.go b/pkg/signal/handler.go
index 0ff833f..2fe89a2 100644
--- a/pkg/signal/handler.go
+++ b/pkg/signal/handler.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package signal implements a handler to listen to system signals.
package signal
import (
@@ -39,6 +40,7 @@ type Handler struct {
cancel chan struct{}
}
+// Name shows the handler's name.
func (h *Handler) Name() string {
return "signal"
}
diff --git a/pkg/convert/hash.go b/pkg/test/doc.go
similarity index 76%
copy from pkg/convert/hash.go
copy to pkg/test/doc.go
index b08c2a5..73c15c4 100644
--- a/pkg/convert/hash.go
+++ b/pkg/test/doc.go
@@ -6,7 +6,7 @@
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
@@ -15,14 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-package convert
-
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
+// Package test implements helpers for testing.
+package test
diff --git a/pkg/test/flags/flags.go b/pkg/test/flags/flags.go
index b2f83f5..92f3790 100644
--- a/pkg/test/flags/flags.go
+++ b/pkg/test/flags/flags.go
@@ -14,6 +14,8 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
+// Package flags implements injected flags to run tests.
package flags
import (
@@ -22,8 +24,12 @@ import (
var (
eventuallyTimeout string
+
+ // EventuallyTimeout is the timeout of async time cases execution.
EventuallyTimeout time.Duration
- LogLevel = "debug"
+
+ // LogLevel is the log level of test cases.
+ LogLevel = "debug"
)
func init() {
diff --git a/pkg/test/flow/slice.go b/pkg/test/flow/slice.go
index 9d6a4fd..984e0f9 100644
--- a/pkg/test/flow/slice.go
+++ b/pkg/test/flow/slice.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package flow implements mockers for testing flow.
package flow
import (
@@ -67,7 +68,7 @@ func (s *sourceSlice) run(ctx context.Context, sliceVal reflect.Value) {
}
}
-func (s *sourceSlice) Teardown(ctx context.Context) error {
+func (s *sourceSlice) Teardown(_ context.Context) error {
s.Wait()
return nil
}
@@ -77,6 +78,7 @@ func (s *sourceSlice) Exec(downstream flow.Inlet) {
go flow.Transmit(&s.ComponentState, downstream, s)
}
+// NewSlice return a new mocked flow.Source.
func NewSlice(slice interface{}) flow.Source {
return &sourceSlice{
slice: slice,
diff --git a/pkg/test/helpers/context.go b/pkg/test/helpers/context.go
index b283903..2b6be79 100644
--- a/pkg/test/helpers/context.go
+++ b/pkg/test/helpers/context.go
@@ -30,11 +30,13 @@ import (
modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
)
+// SharedContext is the context shared between test cases in the integration testing.
type SharedContext struct {
Connection *grpclib.ClientConn
BaseTime time.Time
}
+// Args is a wrapper seals all necessary info for table specs.
type Args struct {
Begin *timestamppb.Timestamp
End *timestamppb.Timestamp
@@ -46,12 +48,14 @@ type Args struct {
WantErr bool
}
+// UnmarshalYAML decodes YAML raw bytes to proto.Message.
func UnmarshalYAML(ii []byte, m proto.Message) {
j, err := yaml.YAMLToJSON(ii)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(protojson.Unmarshal(j, m)).To(gomega.Succeed())
}
+// TimeRange returns a modelv1.TimeRange based on Args and SharedContext.
func TimeRange(args Args, shardContext SharedContext) *modelv1.TimeRange {
if args.Begin != nil && args.End != nil {
return &modelv1.TimeRange{
diff --git a/pkg/convert/hash.go b/pkg/test/helpers/doc.go
similarity index 76%
copy from pkg/convert/hash.go
copy to pkg/test/helpers/doc.go
index b08c2a5..1da300b 100644
--- a/pkg/convert/hash.go
+++ b/pkg/test/helpers/doc.go
@@ -6,7 +6,7 @@
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
@@ -15,14 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-package convert
-
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
+// Package helpers implements several helpers for testing.
+package helpers
diff --git a/pkg/test/helpers/fail_interceptor.go b/pkg/test/helpers/fail_interceptor.go
deleted file mode 100644
index b971088..0000000
--- a/pkg/test/helpers/fail_interceptor.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package helpers
-
-import (
- "sync/atomic"
-
- "github.com/onsi/gomega/types"
-)
-
-type FailInterceptor struct {
- ginkgoFail types.GomegaFailHandler
- didFail *atomic.Bool
-}
-
-func NewFailInterceptor(fail types.GomegaFailHandler) *FailInterceptor {
- return &FailInterceptor{
- ginkgoFail: fail,
- didFail: &atomic.Bool{},
- }
-}
-
-func (f *FailInterceptor) Fail(message string, callerSkip ...int) {
- f.didFail.Store(true)
- if len(callerSkip) == 0 {
- f.ginkgoFail(message, 1)
- } else {
- f.ginkgoFail(message, callerSkip[0]+1)
- }
-}
-
-func (f *FailInterceptor) Reset() {
- f.didFail.Store(false)
-}
-
-func (f *FailInterceptor) DidFail() bool {
- return f.didFail.Load()
-}
diff --git a/pkg/test/helpers/grpc_health.go b/pkg/test/helpers/grpc_health.go
index db46c7d..4b9274c 100644
--- a/pkg/test/helpers/grpc_health.go
+++ b/pkg/test/helpers/grpc_health.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package helpers
import (
@@ -28,8 +29,9 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
-var ErrServiceUnhealthy = errors.New("service is unhealthy")
+var errServiceUnhealthy = errors.New("service is unhealthy")
+// HealthCheck returns a function for ginkgo "Eventually" poll it repeatedly to check whether a gRPC server is ready.
func HealthCheck(addr string, connTimeout time.Duration, rpcTimeout time.Duration, opts ...grpc.DialOption) func() error {
return func() error {
conn, err := grpchelper.Conn(addr, connTimeout, opts...)
@@ -50,7 +52,7 @@ func HealthCheck(addr string, connTimeout time.Duration, rpcTimeout time.Duratio
l := logger.GetLogger()
if resp.GetStatus() != grpc_health_v1.HealthCheckResponse_SERVING {
l.Warn().Str("responded_status", resp.GetStatus().String()).Msg("service unhealthy")
- return ErrServiceUnhealthy
+ return errServiceUnhealthy
}
if e := l.Debug(); e.Enabled() {
e.Stringer("status", resp.GetStatus()).Msg("connected")
diff --git a/pkg/test/helpers/http_health.go b/pkg/test/helpers/http_health.go
index d591be4..365338c 100644
--- a/pkg/test/helpers/http_health.go
+++ b/pkg/test/helpers/http_health.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package helpers
import (
@@ -25,6 +26,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
+// HTTPHealthCheck returns a function for ginkgo "Eventually" poll it repeatedly to check whether a HTTP server is ready.
func HTTPHealthCheck(addr string) func() error {
return func() error {
client := resty.New()
@@ -40,7 +42,7 @@ func HTTPHealthCheck(addr string) func() error {
if resp.StatusCode() != 200 {
l.Warn().Str("responded_status", resp.Status()).Msg("service unhealthy")
time.Sleep(1 * time.Second)
- return ErrServiceUnhealthy
+ return errServiceUnhealthy
}
if e := l.Debug(); e.Enabled() {
e.Stringer("response", resp).Msg("connected")
diff --git a/pkg/test/matcher.go b/pkg/test/matcher.go
index 2a0a0dc..83ac1fe 100644
--- a/pkg/test/matcher.go
+++ b/pkg/test/matcher.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package test
import (
@@ -34,6 +35,7 @@ type shardEventMatcher struct {
action databasev1.Action
}
+// NewShardEventMatcher return a Matcher to comparing shard event's action.
func NewShardEventMatcher(action databasev1.Action) gomock.Matcher {
return &shardEventMatcher{
action: action,
@@ -58,6 +60,7 @@ type entityEventMatcher struct {
action databasev1.Action
}
+// NewEntityEventMatcher return a Matcher for comparing entity event's action.
func NewEntityEventMatcher(action databasev1.Action) gomock.Matcher {
return &entityEventMatcher{
action: action,
diff --git a/pkg/test/measure/etcd.go b/pkg/test/measure/etcd.go
index 8fffc11..8abe58b 100644
--- a/pkg/test/measure/etcd.go
+++ b/pkg/test/measure/etcd.go
@@ -15,29 +15,21 @@
// specific language governing permissions and limitations
// under the License.
+// Package measure implements helpers to load schemas for testing.
package measure
import (
"context"
"embed"
- "fmt"
- "os"
"path"
- "strconv"
- "time"
- "github.com/google/uuid"
"github.com/pkg/errors"
- "go.uber.org/multierr"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
- "github.com/apache/skywalking-banyandb/pkg/grpchelper"
)
const (
@@ -51,6 +43,7 @@ const (
//go:embed testdata/*
var store embed.FS
+// PreloadSchema loads schemas from files in the booting process.
func PreloadSchema(e schema.Registry) error {
if err := loadSchema(groupDir, &commonv1.Group{}, func(group *commonv1.Group) error {
return e.CreateGroup(context.TODO(), group)
@@ -80,10 +73,6 @@ func PreloadSchema(e schema.Registry) error {
return nil
}
-func RandomTempDir() string {
- return path.Join(os.TempDir(), fmt.Sprintf("banyandb-embed-etcd-%s", uuid.New().String()))
-}
-
func loadSchema[T proto.Message](dir string, resource T, loadFn func(resource T) error) error {
entries, err := store.ReadDir(dir)
if err != nil {
@@ -104,76 +93,3 @@ func loadSchema[T proto.Message](dir string, resource T, loadFn func(resource T)
}
return nil
}
-
-var rpcTimeout = 10 * time.Second
-
-func RegisterForNew(addr string, metricNum int) error {
- conn, err := grpchelper.Conn(addr, 1*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
- if err != nil {
- return err
- }
- defer conn.Close()
-
- ctx := context.Background()
-
- if err := loadSchema(groupDir, &commonv1.Group{}, func(group *commonv1.Group) error {
- return grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewGroupRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.GroupRegistryServiceCreateRequest{
- Group: group,
- })
- return err
- })
- }); err != nil {
- return errors.WithStack(err)
- }
- if err := loadSchema(measureDir, &databasev1.Measure{}, func(measure *databasev1.Measure) error {
- var err error
- name := measure.GetMetadata().GetName()
- num := metricNum
- if name != "service_cpm_minute" {
- num = 1
- }
- for i := 0; i < num; i++ {
- err = multierr.Append(err, grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- m := proto.Clone(measure).(*databasev1.Measure)
- if i > 0 {
- m.Metadata.Name = m.GetMetadata().GetName() + "_" + strconv.Itoa(i)
- }
- _, err = databasev1.NewMeasureRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.MeasureRegistryServiceCreateRequest{
- Measure: m,
- })
- return err
- }))
- }
- return err
- }); err != nil {
- return errors.WithStack(err)
- }
-
- if err := loadSchema(indexRuleDir, &databasev1.IndexRule{}, func(indexRule *databasev1.IndexRule) error {
- return grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewIndexRuleRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.IndexRuleRegistryServiceCreateRequest{
- IndexRule: indexRule,
- })
- return err
- })
- }); err != nil {
- return errors.WithStack(err)
- }
- if err := loadSchema(indexRuleBindingDir, &databasev1.IndexRuleBinding{}, func(indexRuleBinding *databasev1.IndexRuleBinding) error {
- return grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewIndexRuleBindingRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
- IndexRuleBinding: indexRuleBinding,
- })
- return err
- })
- }); err != nil {
- return errors.WithStack(err)
- }
-
- return nil
-}
diff --git a/pkg/test/measure/traffic/traffic.go b/pkg/test/measure/traffic/traffic.go
deleted file mode 100644
index 0b40479..0000000
--- a/pkg/test/measure/traffic/traffic.go
+++ /dev/null
@@ -1,187 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package traffic
-
-import (
- "context"
- "crypto/rand"
- "errors"
- "fmt"
- "io"
- "strconv"
- "time"
-
- "github.com/dgraph-io/ristretto/z"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
- "google.golang.org/protobuf/types/known/timestamppb"
-
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- "github.com/apache/skywalking-banyandb/pkg/grpchelper"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var l = logger.GetLogger("test_measure_traffic")
-
-type TestCase struct {
- Addr string
- SvcNum int
- InstanceNumEverySvc int
- MetricNum int
-}
-
-func SendWrites(ts TestCase) (*z.Closer, error) {
- conn, err := grpchelper.Conn(ts.Addr, 1*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
- if err != nil {
- return nil, err
- }
- client := measurev1.NewMeasureServiceClient(conn)
- closer := z.NewCloser(0)
- for i := 0; i < ts.SvcNum; i++ {
- for j := 0; j < ts.InstanceNumEverySvc; j++ {
- data := make([]byte, 10*1024)
- _, _ = rand.Read(data)
- sender, err := newSender(i, j, client, ts.MetricNum)
- if err != nil {
- l.Err(err).Msg("failed to create a new sender")
- return nil, err
- }
- closer.AddRunning(1)
- go func() {
- ticker := time.NewTicker(1 * time.Minute)
- for {
- select {
- case <-ticker.C:
- sender.write()
- l.Info().Msg(sender.String())
- case <-closer.HasBeenClosed():
- ticker.Stop()
- closer.Done()
- return
- }
- }
- }()
- }
- }
- go func() {
- closer.Wait()
- conn.Close()
- }()
- return closer, nil
-}
-
-type sender struct {
- client measurev1.MeasureService_WriteClient
- svc string
- instance string
- succeed uint64
- sendFailed uint64
- revFailed uint64
- metricNum int
-}
-
-func newSender(svcID, instanceID int, client measurev1.MeasureServiceClient, metricNum int) (*sender, error) {
- wc, err := client.Write(context.Background())
- if err != nil {
- l.Error().Err(err).Msg("creating write client failed")
- return nil, err
- }
- return &sender{
- svc: "svc-" + strconv.Itoa(svcID),
- instance: "instance-" + strconv.Itoa(instanceID),
- client: wc,
- metricNum: metricNum,
- }, nil
-}
-
-func (s *sender) write() {
- ts := timestamppb.New(timestamp.NowMilli())
- dp := &measurev1.DataPointValue{
- Timestamp: ts,
- TagFamilies: []*modelv1.TagFamilyForWrite{
- {
- Tags: []*modelv1.TagValue{
- {
- Value: &modelv1.TagValue_Id{
- Id: &modelv1.ID{
- Value: s.instance,
- },
- },
- },
- {
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: s.svc,
- },
- },
- },
- },
- },
- },
- Fields: []*modelv1.FieldValue{
- {
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: 10,
- },
- },
- },
- {
- Value: &modelv1.FieldValue_Int{
- Int: &modelv1.Int{
- Value: 100,
- },
- },
- },
- },
- }
- for i := 0; i < s.metricNum; i++ {
- name := "service_cpm_minute"
- if i > 0 {
- name = name + "_" + strconv.Itoa(i)
- }
- err := s.client.Send(&measurev1.WriteRequest{
- Metadata: &commonv1.Metadata{
- Name: name,
- Group: "sw_metric",
- },
-
- DataPoint: dp,
- })
- if err != nil {
- l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("writing failed")
- s.sendFailed++
- return
- }
- _, err = s.client.Recv()
- if errors.Is(err, io.EOF) {
- l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("receiving failed")
- s.revFailed++
- return
- }
- s.succeed++
- }
-}
-
-func (s *sender) String() string {
- return fmt.Sprintf("%s %s writing succeed %d failed %d(sending) %d(receiving)",
- s.svc, s.instance, s.succeed, s.sendFailed, s.revFailed)
-}
diff --git a/pkg/test/network.go b/pkg/test/network.go
index 6536b6b..cc4ad36 100644
--- a/pkg/test/network.go
+++ b/pkg/test/network.go
@@ -46,6 +46,7 @@ func AllocateFreePorts(n int) ([]int, error) {
return ports, nil
}
+// NewEtcdListenUrls returns two urls with different ports.
func NewEtcdListenUrls() (string, string, error) {
ports, err := AllocateFreePorts(2)
if err != nil {
diff --git a/pkg/test/setup.go b/pkg/test/setup.go
index 829b67e..a684b64 100644
--- a/pkg/test/setup.go
+++ b/pkg/test/setup.go
@@ -25,8 +25,9 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-func SetUpModules(flags []string, units ...run.Unit) func() {
- closer := run.NewTester("closer")
+// SetupModules wires input modules to build a testing ready runtime.
+func SetupModules(flags []string, units ...run.Unit) func() {
+ closer, deferFn := run.NewTester("closer")
g := run.NewGroup("standalone-test")
g.Register(append([]run.Unit{closer}, units...)...)
err := g.RegisterFlags().Parse(flags)
@@ -44,7 +45,7 @@ func SetUpModules(flags []string, units ...run.Unit) func() {
}()
g.WaitTillReady()
return func() {
- closer.GracefulStop()
+ deferFn()
wg.Wait()
}
}
diff --git a/pkg/test/setup/setup.go b/pkg/test/setup/setup.go
index e8b3063..e172455 100644
--- a/pkg/test/setup/setup.go
+++ b/pkg/test/setup/setup.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package setup implements a real env in which to run tests.
package setup
import (
@@ -38,7 +39,8 @@ import (
const host = "127.0.0.1"
-func SetUp(flags ...string) (string, string, func()) {
+// Common wires common modules to build a testing ready runtime.
+func Common(flags ...string) (string, string, func()) {
path, deferFn, err := test.NewSpace()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
var ports []int
@@ -87,7 +89,7 @@ func modules(flags []string) func() {
tcp := grpc.NewServer(context.TODO(), pipeline, repo, metaSvc)
httpServer := http.NewService()
- return test.SetUpModules(
+ return test.SetupModules(
flags,
repo,
pipeline,
diff --git a/pkg/test/space.go b/pkg/test/space.go
index 9e9e64a..14448be 100644
--- a/pkg/test/space.go
+++ b/pkg/test/space.go
@@ -24,6 +24,8 @@ import (
"github.com/stretchr/testify/require"
)
+// Space create a tmp dir and returns the path and its clear function.
+// It will be panic when encountering some errors.
func Space(t *require.Assertions) (tempDir string, deferFunc func()) {
var tempDirErr error
tempDir, tempDirErr = os.MkdirTemp("", "banyandb-test-*")
@@ -35,6 +37,8 @@ func Space(t *require.Assertions) (tempDir string, deferFunc func()) {
}
}
+// NewSpace create a tmp dir and returns the path and its clear function.
+// It will return any error when failed to create this dir.
func NewSpace() (tempDir string, deferFunc func(), err error) {
tempDir, err = os.MkdirTemp("", "banyandb-test-*")
return tempDir, func() {
diff --git a/pkg/test/stream/etcd.go b/pkg/test/stream/etcd.go
index 1f1ba8a..36ad948 100644
--- a/pkg/test/stream/etcd.go
+++ b/pkg/test/stream/etcd.go
@@ -15,25 +15,18 @@
// specific language governing permissions and limitations
// under the License.
+// Package stream implements helpers to load schemas for testing.
package stream
import (
"context"
"embed"
- "fmt"
- "os"
- "path"
- "time"
- "github.com/google/uuid"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/encoding/protojson"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
- "github.com/apache/skywalking-banyandb/pkg/grpchelper"
)
const indexRuleDir = "testdata/index_rules"
@@ -49,6 +42,7 @@ var (
groupJSON string
)
+// PreloadSchema loads schemas from files in the booting process.
func PreloadSchema(e schema.Registry) error {
g := &commonv1.Group{}
if err := protojson.Unmarshal([]byte(groupJSON), g); err != nil {
@@ -95,86 +89,3 @@ func PreloadSchema(e schema.Registry) error {
return nil
}
-
-func RandomTempDir() string {
- return path.Join(os.TempDir(), fmt.Sprintf("banyandb-embed-etcd-%s", uuid.New().String()))
-}
-
-var rpcTimeout = 10 * time.Second
-
-func RegisterForNew(addr string) error {
- conn, err := grpchelper.Conn(addr, 1*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
- if err != nil {
- return err
- }
- defer conn.Close()
-
- ctx := context.Background()
- g := &commonv1.Group{}
- if err = protojson.Unmarshal([]byte(groupJSON), g); err != nil {
- return err
- }
- if err = grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewGroupRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.GroupRegistryServiceCreateRequest{
- Group: g,
- })
- return err
- }); err != nil {
- return err
- }
-
- s := &databasev1.Stream{}
- if unmarshalErr := protojson.Unmarshal([]byte(streamJSON), s); unmarshalErr != nil {
- return unmarshalErr
- }
- if err = grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewStreamRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.StreamRegistryServiceCreateRequest{
- Stream: s,
- })
- return err
- }); err != nil {
- return err
- }
-
- indexRuleBinding := &databasev1.IndexRuleBinding{}
- if unmarshalErr := protojson.Unmarshal([]byte(indexRuleBindingJSON), indexRuleBinding); unmarshalErr != nil {
- return unmarshalErr
- }
- if err = grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewIndexRuleBindingRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.IndexRuleBindingRegistryServiceCreateRequest{
- IndexRuleBinding: indexRuleBinding,
- })
- return err
- }); err != nil {
- return err
- }
-
- entries, err := indexRuleStore.ReadDir(indexRuleDir)
- if err != nil {
- return err
- }
- for _, entry := range entries {
- data, err := indexRuleStore.ReadFile(indexRuleDir + "/" + entry.Name())
- if err != nil {
- return err
- }
- idxRule := &databasev1.IndexRule{}
- err = protojson.Unmarshal(data, idxRule)
- if err != nil {
- return err
- }
- if err = grpchelper.Request(ctx, rpcTimeout, func(rpcCtx context.Context) (err error) {
- _, err = databasev1.NewIndexRuleRegistryServiceClient(conn).
- Create(rpcCtx, &databasev1.IndexRuleRegistryServiceCreateRequest{
- IndexRule: idxRule,
- })
- return err
- }); err != nil {
- return err
- }
- }
- return nil
-}
diff --git a/pkg/test/stream/traffic/searchable_template.json b/pkg/test/stream/traffic/searchable_template.json
deleted file mode 100644
index f50a757..0000000
--- a/pkg/test/stream/traffic/searchable_template.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "tags": [
- {"str":{"value": "1"}},
- {"int":{"value": 0}},
- {"str":{"value": "webapp_id"}},
- {"str":{"value": "10.0.0.1_id"}},
- {"str":{"value": "/home_id"}},
- {"int":{"value": 1000}},
- {"int":{"value": 1622933202000000000}}
- ]
-}
-
\ No newline at end of file
diff --git a/pkg/test/stream/traffic/traffic.go b/pkg/test/stream/traffic/traffic.go
deleted file mode 100644
index 7ffe143..0000000
--- a/pkg/test/stream/traffic/traffic.go
+++ /dev/null
@@ -1,195 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package traffic
-
-import (
- "context"
- "crypto/rand"
- // Load some tag templates.
- _ "embed"
- "encoding/base64"
- "errors"
- "fmt"
- "io"
- "strconv"
- "strings"
- "time"
-
- "github.com/dgraph-io/ristretto/z"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
- "google.golang.org/protobuf/encoding/protojson"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/timestamppb"
-
- commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
- streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
- "github.com/apache/skywalking-banyandb/pkg/grpchelper"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
-)
-
-var (
- //go:embed searchable_template.json
- content []byte
- l = logger.GetLogger("test_stream_traffic")
-)
-
-type TestCase struct {
- Addr string
- SvcNum int
- InstanceNumEverySvc int
-}
-
-func SendWrites(ts TestCase) (*z.Closer, error) {
- searchTagFamily := &modelv1.TagFamilyForWrite{}
- err := protojson.Unmarshal(content, searchTagFamily)
- if err != nil {
- l.Err(err).Msg("unmarshal template")
- return nil, err
- }
- conn, err := grpchelper.Conn(ts.Addr, 1*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
- if err != nil {
- return nil, err
- }
- client := streamv1.NewStreamServiceClient(conn)
- closer := z.NewCloser(0)
- for i := 0; i < ts.SvcNum; i++ {
- for j := 0; j < ts.InstanceNumEverySvc; j++ {
- data := make([]byte, 10*1024)
- _, _ = rand.Read(data)
- sender, err := newSender(i, j, client, data)
- if err != nil {
- l.Err(err).Msg("failed to create a new sender")
- return nil, err
- }
- closer.AddRunning(1)
- go func() {
- ticker := time.NewTicker(1 * time.Second)
- step := 0
- for {
- select {
- case <-ticker.C:
- sender.write(searchTagFamily)
- step++
- if step%10 == 0 {
- l.Info().Msg(sender.String())
- }
- case <-closer.HasBeenClosed():
- ticker.Stop()
- closer.Done()
- return
- }
- }
- }()
- }
- }
- go func() {
- closer.Wait()
- conn.Close()
- }()
- return closer, nil
-}
-
-type sender struct {
- client streamv1.StreamService_WriteClient
- svc string
- instance string
- data []byte
- succeed uint64
- sendFailed uint64
- revFailed uint64
-}
-
-func newSender(svcID, instanceID int, client streamv1.StreamServiceClient, data []byte) (*sender, error) {
- wc, err := client.Write(context.Background())
- if err != nil {
- l.Error().Err(err).Msg("creating write client failed")
- return nil, err
- }
- return &sender{
- svc: "svc-" + strconv.Itoa(svcID),
- instance: "instance-" + strconv.Itoa(instanceID),
- client: wc,
- data: data,
- }, nil
-}
-
-func (s *sender) write(searchTagFamily *modelv1.TagFamilyForWrite) {
- tf := proto.Clone(searchTagFamily).(*modelv1.TagFamilyForWrite)
- tf.Tags[2] = &modelv1.TagValue{
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: s.svc,
- },
- },
- }
- tf.Tags[3] = &modelv1.TagValue{
- Value: &modelv1.TagValue_Str{
- Str: &modelv1.Str{
- Value: s.instance,
- },
- },
- }
-
- ts := timestamppb.New(timestamp.NowMilli())
-
- eleID := base64.StdEncoding.EncodeToString([]byte(strings.Join(
- []string{s.svc, s.instance, strconv.Itoa(int(ts.AsTime().UnixMilli()))}, "-")))
- e := &streamv1.ElementValue{
- ElementId: eleID,
- Timestamp: ts,
- TagFamilies: []*modelv1.TagFamilyForWrite{
- {
- Tags: []*modelv1.TagValue{
- {
- Value: &modelv1.TagValue_BinaryData{
- BinaryData: s.data,
- },
- },
- },
- },
- },
- }
- e.TagFamilies = append(e.TagFamilies, tf)
- err := s.client.Send(&streamv1.WriteRequest{
- Metadata: &commonv1.Metadata{
- Name: "sw",
- Group: "default",
- },
- Element: e,
- })
- if err != nil {
- l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("writing failed")
- s.sendFailed++
- return
- }
- _, err = s.client.Recv()
- if errors.Is(err, io.EOF) {
- l.Error().Str("svc", s.svc).Str("instance", s.instance).Err(err).Msg("receiving failed")
- s.revFailed++
- return
- }
- s.succeed++
-}
-
-func (s *sender) String() string {
- return fmt.Sprintf("%s %s writing succeed %d failed %d(sending) %d(receiving)",
- s.svc, s.instance, s.succeed, s.sendFailed, s.revFailed)
-}
diff --git a/pkg/timestamp/clock.go b/pkg/timestamp/clock.go
index 1746e2a..152e106 100644
--- a/pkg/timestamp/clock.go
+++ b/pkg/timestamp/clock.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package timestamp
import (
@@ -28,6 +29,7 @@ type Clock interface {
clock.Clock
}
+// MockClock represents a mock clock that only moves forward programmatically.
type MockClock interface {
clock.Clock
// Add moves the current time of the mock clock forward by the specified duration.
@@ -64,7 +66,7 @@ func GetClock(ctx context.Context) (Clock, context.Context) {
return c.(Clock), ctx
}
-// SetContext returns a sub context with the passed Clock.
+// SetClock returns a sub context with the passed Clock.
func SetClock(ctx context.Context, clock Clock) context.Context {
return context.WithValue(ctx, clockKey, clock)
}
diff --git a/pkg/convert/hash.go b/pkg/timestamp/doc.go
similarity index 81%
copy from pkg/convert/hash.go
copy to pkg/timestamp/doc.go
index b08c2a5..f025925 100644
--- a/pkg/convert/hash.go
+++ b/pkg/timestamp/doc.go
@@ -15,14 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-package convert
-
-import "github.com/cespare/xxhash"
-
-func Hash(key []byte) uint64 {
- return xxhash.Sum64(key)
-}
-
-func HashStr(key string) uint64 {
- return xxhash.Sum64String(key)
-}
+// Package timestamp implements several component related to time.
+package timestamp
diff --git a/pkg/timestamp/nano.go b/pkg/timestamp/nano.go
index bdd3bbb..3693754 100644
--- a/pkg/timestamp/nano.go
+++ b/pkg/timestamp/nano.go
@@ -70,26 +70,28 @@ var (
minNanoTime = time.Unix(0, MinNanoTime).UTC()
maxNanoTime = time.Unix(0, MaxNanoTime).UTC()
- MaxMilliTime = time.UnixMilli(maxNanoTime.UnixMilli())
- MaxMilliPbTime = timestamppb.New(MaxMilliTime)
- DefaultBeginPbTime = timestamppb.New(time.Unix(0, 0))
- DefaultTimeRange = &modelv1.TimeRange{
- Begin: DefaultBeginPbTime,
- End: MaxMilliPbTime,
+ maxMilliTime = time.UnixMilli(maxNanoTime.UnixMilli())
+ maxMilliPbTime = timestamppb.New(maxMilliTime)
+ defaultBeginPbTime = timestamppb.New(time.Unix(0, 0))
+
+ // DefaultTimeRange for the input time range.
+ DefaultTimeRange = &modelv1.TimeRange{
+ Begin: defaultBeginPbTime,
+ End: maxMilliPbTime,
}
- ErrTimeOutOfRange = errors.Errorf("time is out of range %d - %d", MinNanoTime, MaxNanoTime)
- ErrTimeNotMillisecond = errors.Errorf("time is not millisecond precision")
- ErrTimeEmpty = errors.Errorf("time is empty")
+ errTimeOutOfRange = errors.Errorf("time is out of range %d - %d", MinNanoTime, MaxNanoTime)
+ errTimeNotMillisecond = errors.Errorf("time is not millisecond precision")
+ errTimeEmpty = errors.Errorf("time is empty")
)
// Check checks that a time is valid.
func Check(t time.Time) error {
if t.Before(minNanoTime) || t.After(maxNanoTime) {
- return ErrTimeOutOfRange
+ return errTimeOutOfRange
}
if t.Nanosecond()%int(mSecond) > 0 {
- return ErrTimeNotMillisecond
+ return errTimeNotMillisecond
}
return nil
}
@@ -97,7 +99,7 @@ func Check(t time.Time) error {
// CheckPb checks that a protobuf timestamp is valid.
func CheckPb(t *timestamppb.Timestamp) error {
if t == nil {
- return ErrTimeEmpty
+ return errTimeEmpty
}
return Check(t.AsTime())
}
@@ -105,7 +107,7 @@ func CheckPb(t *timestamppb.Timestamp) error {
// CheckTimeRange checks that a protobuf time range is valid.
func CheckTimeRange(timeRange *modelv1.TimeRange) error {
if timeRange == nil {
- return ErrTimeEmpty
+ return errTimeEmpty
}
err := CheckPb(timeRange.Begin)
if err != nil {
diff --git a/pkg/timestamp/range.go b/pkg/timestamp/range.go
index 46ac360..0376187 100644
--- a/pkg/timestamp/range.go
+++ b/pkg/timestamp/range.go
@@ -21,6 +21,7 @@ import (
"time"
)
+// TimeRange is a range of periods into which data can be written or retrieved.
type TimeRange struct {
Start time.Time
End time.Time
@@ -28,6 +29,7 @@ type TimeRange struct {
IncludeEnd bool
}
+// Contains returns whether the unixNano is in the TimeRange.
func (t TimeRange) Contains(unixNano uint64) bool {
tp := time.Unix(0, int64(unixNano))
if t.Start.Equal(tp) {
@@ -39,6 +41,7 @@ func (t TimeRange) Contains(unixNano uint64) bool {
return !tp.Before(t.Start) && !tp.After(t.End)
}
+// Overlapping returns whether TimeRanges intersect each other.
func (t TimeRange) Overlapping(other TimeRange) bool {
if t.Start.Equal(other.End) {
return t.IncludeStart && other.IncludeEnd
@@ -49,10 +52,12 @@ func (t TimeRange) Overlapping(other TimeRange) bool {
return !t.Start.After(other.End) && !other.Start.After(t.End)
}
+// Duration converts TimeRange to time.Duration.
func (t TimeRange) Duration() time.Duration {
return t.End.Sub(t.Start)
}
+// String shows the string representation.
func (t TimeRange) String() string {
var buf []byte
if t.IncludeStart {
@@ -71,6 +76,7 @@ func (t TimeRange) String() string {
return string(buf)
}
+// NewInclusiveTimeRange returns TimeRange includes start and end time.
func NewInclusiveTimeRange(start, end time.Time) TimeRange {
return TimeRange{
Start: start,
@@ -80,14 +86,19 @@ func NewInclusiveTimeRange(start, end time.Time) TimeRange {
}
}
+// NewInclusiveTimeRangeDuration returns TimeRange includes start and end time.
+// It is created from a start time and the Duration.
func NewInclusiveTimeRangeDuration(start time.Time, duration time.Duration) TimeRange {
return NewTimeRangeDuration(start, duration, true, true)
}
+// NewSectionTimeRange returns TimeRange includes start time.
+// This function is for creating blocks and segments of tsdb.
func NewSectionTimeRange(start, end time.Time) TimeRange {
return NewTimeRange(start, end, true, false)
}
+// NewTimeRange returns TimeRange.
func NewTimeRange(start, end time.Time, includeStart, includeEnd bool) TimeRange {
return TimeRange{
Start: start,
@@ -97,6 +108,7 @@ func NewTimeRange(start, end time.Time, includeStart, includeEnd bool) TimeRange
}
}
+// NewTimeRangeDuration returns TimeRange from a start time and the Duration.
func NewTimeRangeDuration(start time.Time, duration time.Duration, includeStart, includeEnd bool) TimeRange {
return NewTimeRange(start, start.Add(duration), includeStart, includeEnd)
}
diff --git a/pkg/timestamp/scheduler.go b/pkg/timestamp/scheduler.go
index 15e0e38..0c85e9c 100644
--- a/pkg/timestamp/scheduler.go
+++ b/pkg/timestamp/scheduler.go
@@ -30,8 +30,11 @@ import (
)
var (
+ // ErrSchedulerClosed indicates the scheduler is closed.
ErrSchedulerClosed = errors.New("the scheduler is closed")
- ErrTaskDuplicated = errors.New("the task is duplicated")
+
+ // ErrTaskDuplicated indicates registered task already exists.
+ ErrTaskDuplicated = errors.New("the task is duplicated")
)
// SchedulerAction is an executable when a trigger is fired
diff --git a/pkg/version/version.go b/pkg/version/version.go
index 21827cb..33969f0 100644
--- a/pkg/version/version.go
+++ b/pkg/version/version.go
@@ -27,7 +27,7 @@ import (
// build is to be populated at build time using -ldflags -X.
var build string
-// Show the service's build information.
+// Build shows the service's build information.
func Build() string {
return build
}
diff --git a/scripts/build/lint.mk b/scripts/build/lint.mk
index 453d2d3..0899dfc 100644
--- a/scripts/build/lint.mk
+++ b/scripts/build/lint.mk
@@ -24,7 +24,7 @@ include $(mk_dir)lint-bin.mk
.PHONY: lint
lint: $(LINTER) $(REVIVE) ## Run all linters
- $(LINTER) run -v --config $(root_dir)/.golangci.yml ./... && \
+ $(LINTER) run -v --config $(root_dir)/.golangci.yml --timeout 10m ./... && \
$(REVIVE) -config $(root_dir)/revive.toml -formatter friendly ./...
.PHONY: format
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index 4ac8cde..058cd55 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -46,7 +46,7 @@ var _ = g.DescribeTable("Scanning Streams", func(args helpers.Args) {
verify(innerGm, args)
}, flags.EventuallyTimeout).Should(gm.Succeed())
},
- g.FEntry("all elements", helpers.Args{Input: "all", Duration: 1 * time.Hour}),
+ g.Entry("all elements", helpers.Args{Input: "all", Duration: 1 * time.Hour}),
g.Entry("limit", helpers.Args{Input: "limit", Duration: 1 * time.Hour}),
g.Entry("offset", helpers.Args{Input: "offset", Duration: 1 * time.Hour}),
g.Entry("nothing", helpers.Args{Input: "all", WantEmpty: true}),
diff --git a/test/integration/cold_query/query_suite_test.go b/test/integration/cold_query/query_suite_test.go
index 17c4df5..4c2e9f3 100644
--- a/test/integration/cold_query/query_suite_test.go
+++ b/test/integration/cold_query/query_suite_test.go
@@ -56,7 +56,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
Level: flags.LogLevel,
})).To(Succeed())
var addr string
- addr, _, deferFunc = setup.SetUp()
+ addr, _, deferFunc = setup.Common()
Eventually(
helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(Succeed())
diff --git a/test/integration/load/load_suite_test.go b/test/integration/load/load_suite_test.go
index 3f92925..1fb5639 100644
--- a/test/integration/load/load_suite_test.go
+++ b/test/integration/load/load_suite_test.go
@@ -52,7 +52,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
Level: "warn",
})).To(Succeed())
var addr string
- addr, _, deferFunc = setup.SetUp()
+ addr, _, deferFunc = setup.Common()
Eventually(
helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(Succeed())
diff --git a/test/integration/other/measure_test.go b/test/integration/other/measure_test.go
index 36bcd8a..830b503 100644
--- a/test/integration/other/measure_test.go
+++ b/test/integration/other/measure_test.go
@@ -41,7 +41,7 @@ var _ = g.Describe("Query service_cpm_minute", func() {
g.BeforeEach(func() {
var addr string
- addr, _, deferFn = setup.SetUp()
+ addr, _, deferFn = setup.Common()
gm.Eventually(helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(gm.Succeed())
var err error
diff --git a/test/integration/other/property_test.go b/test/integration/other/property_test.go
index fa1f9d2..fcace89 100644
--- a/test/integration/other/property_test.go
+++ b/test/integration/other/property_test.go
@@ -42,7 +42,7 @@ var _ = Describe("Property application", func() {
BeforeEach(func() {
var addr string
- addr, _, deferFn = setup.SetUp()
+ addr, _, deferFn = setup.Common()
Eventually(helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(Succeed())
var err error
@@ -104,7 +104,7 @@ var _ = Describe("Property application", func() {
BeforeEach(func() {
var addr string
- addr, _, deferFn = setup.SetUp()
+ addr, _, deferFn = setup.Common()
var err error
conn, err = grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
Expect(err).NotTo(HaveOccurred())
diff --git a/test/integration/other/tls_test.go b/test/integration/other/tls_test.go
index 8560e39..d0cc3d2 100644
--- a/test/integration/other/tls_test.go
+++ b/test/integration/other/tls_test.go
@@ -47,7 +47,7 @@ var _ = g.Describe("Query service_cpm_minute", func() {
certFile := filepath.Join(basePath, "testdata/server_cert.pem")
keyFile := filepath.Join(basePath, "testdata/server_key.pem")
var addr string
- addr, _, deferFn = setup.SetUp("--tls=true", "--cert-file="+certFile, "--key-file="+keyFile)
+ addr, _, deferFn = setup.Common("--tls=true", "--cert-file="+certFile, "--key-file="+keyFile)
var err error
creds, err := credentials.NewClientTLSFromFile(certFile, "localhost")
gm.Expect(err).NotTo(gm.HaveOccurred())
diff --git a/test/integration/query/query_suite_test.go b/test/integration/query/query_suite_test.go
index e8dc0ff..5861a58 100644
--- a/test/integration/query/query_suite_test.go
+++ b/test/integration/query/query_suite_test.go
@@ -56,7 +56,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
Level: flags.LogLevel,
})).To(Succeed())
var addr string
- addr, _, deferFunc = setup.SetUp()
+ addr, _, deferFunc = setup.Common()
Eventually(helpers.HealthCheck(addr, 10*time.Second, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(Succeed())
conn, err := grpchelper.Conn(addr, 10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials()))
[skywalking-banyandb] 02/03: Fix lint issues in banyand
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch lint
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit c3adea8356fafa4e4c01a443980b619d08373e25
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri Dec 2 07:10:42 2022 +0000
Fix lint issues in banyand
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/cmd/server/main.go | 1 +
banyand/discovery/discovery.go | 4 ++++
banyand/doc.go | 1 +
banyand/internal/cmd/root.go | 1 +
banyand/internal/cmd/standalone.go | 2 +-
banyand/kv/badger.go | 21 +++++++---------
banyand/kv/kv.go | 40 ++++++++++++++++++-------------
banyand/liaison/grpc/discovery.go | 6 ++---
banyand/liaison/grpc/measure.go | 9 +++----
banyand/liaison/grpc/registry.go | 2 +-
banyand/liaison/grpc/server.go | 34 +++++++++++++-------------
banyand/liaison/grpc/stream.go | 7 ++----
banyand/liaison/http/health.go | 4 ++--
banyand/liaison/http/server.go | 14 +++++------
banyand/liaison/liaison.go | 10 +++-----
banyand/measure/field_flag_test.go | 1 +
banyand/measure/measure.go | 7 +++---
banyand/measure/measure_query.go | 6 +++--
banyand/measure/measure_topn.go | 10 ++++----
banyand/measure/measure_write.go | 10 ++++----
banyand/measure/metadata.go | 2 +-
banyand/measure/service.go | 6 +++--
banyand/metadata/metadata.go | 5 ++++
banyand/observability/metric.go | 2 ++
banyand/observability/pprof.go | 1 +
banyand/observability/type.go | 4 ++++
banyand/query/processor.go | 3 ++-
banyand/query/query.go | 10 ++++----
banyand/queue/local.go | 2 +-
banyand/queue/queue.go | 3 +++
banyand/stream/metadata.go | 2 +-
banyand/stream/service.go | 9 +++----
banyand/stream/stream.go | 4 +++-
banyand/stream/stream_query.go | 6 +++--
banyand/stream/stream_write.go | 6 ++---
banyand/tsdb/block.go | 12 +++++-----
banyand/tsdb/block_ctrl.go | 10 ++++----
banyand/tsdb/bucket/bucket.go | 23 +++++++++++-------
banyand/tsdb/bucket/queue.go | 18 +++++++++-----
banyand/tsdb/bucket/strategy.go | 33 +++++++++++++++----------
banyand/tsdb/index/writer.go | 16 ++++++-------
banyand/tsdb/indexdb.go | 14 ++++++-----
banyand/tsdb/metric.go | 1 +
banyand/tsdb/retention.go | 2 +-
banyand/tsdb/scope.go | 21 ++++++++--------
banyand/tsdb/segment.go | 2 +-
banyand/tsdb/segment_ctrl.go | 8 +++----
banyand/tsdb/series.go | 22 ++++++++++-------
banyand/tsdb/series_seek.go | 4 ++++
banyand/tsdb/series_seek_filter.go | 6 ++---
banyand/tsdb/series_seek_sort.go | 4 ++--
banyand/tsdb/series_write.go | 21 ++++++++--------
banyand/tsdb/seriesdb.go | 48 +++++++++++++++++++++++++++++--------
banyand/tsdb/seriesdb_test.go | 2 +-
banyand/tsdb/shard.go | 22 +++++++----------
banyand/tsdb/tsdb.go | 49 +++++++++++++++++++++++++++-----------
pkg/schema/metadata.go | 4 ++--
pkg/test/setup/setup.go | 2 +-
58 files changed, 353 insertions(+), 246 deletions(-)
diff --git a/banyand/cmd/server/main.go b/banyand/cmd/server/main.go
index 6a2568c..8906f84 100644
--- a/banyand/cmd/server/main.go
+++ b/banyand/cmd/server/main.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package main implements the executable banyandb server named banyand.
package main
import (
diff --git a/banyand/discovery/discovery.go b/banyand/discovery/discovery.go
index 34c4cf8..daacd59 100644
--- a/banyand/discovery/discovery.go
+++ b/banyand/discovery/discovery.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package discovery implements the service discovery.
package discovery
import (
@@ -24,6 +25,8 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
+// ServiceRepo provides service subscripting and publishing.
+//
//go:generate mockgen -destination=./discovery_mock.go -package=discovery github.com/apache/skywalking-banyandb/banyand/discovery ServiceRepo
type ServiceRepo interface {
NodeID() string
@@ -52,6 +55,7 @@ func (r *repo) Publish(topic bus.Topic, message ...bus.Message) (bus.Future, err
return r.local.Publish(topic, message...)
}
+// NewServiceRepo return a new ServiceRepo.
func NewServiceRepo(_ context.Context) (ServiceRepo, error) {
return &repo{
local: bus.NewBus(),
diff --git a/banyand/doc.go b/banyand/doc.go
index 94f961d..ab85caf 100644
--- a/banyand/doc.go
+++ b/banyand/doc.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package banyand implements a executable database server.
package banyand
/*
diff --git a/banyand/internal/cmd/root.go b/banyand/internal/cmd/root.go
index 01cc159..c5dd966 100644
--- a/banyand/internal/cmd/root.go
+++ b/banyand/internal/cmd/root.go
@@ -33,6 +33,7 @@ const logo = `
╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═══╝╚═════╝ ╚═════╝
`
+// NewRoot returns a root command.
func NewRoot() *cobra.Command {
cmd := &cobra.Command{
DisableAutoGenTag: true,
diff --git a/banyand/internal/cmd/standalone.go b/banyand/internal/cmd/standalone.go
index 020fb51..2f3b2ad 100644
--- a/banyand/internal/cmd/standalone.go
+++ b/banyand/internal/cmd/standalone.go
@@ -65,7 +65,7 @@ func newStandaloneCmd() *cobra.Command {
if err != nil {
l.Fatal().Err(err).Msg("failed to initiate measure service")
}
- q, err := query.NewExecutor(ctx, streamSvc, measureSvc, metaSvc, repo, pipeline)
+ q, err := query.NewService(ctx, streamSvc, measureSvc, metaSvc, repo, pipeline)
if err != nil {
l.Fatal().Err(err).Msg("failed to initiate query processor")
}
diff --git a/banyand/kv/badger.go b/banyand/kv/badger.go
index 584a664..6fb9c0b 100644
--- a/banyand/kv/badger.go
+++ b/banyand/kv/badger.go
@@ -34,12 +34,13 @@ import (
)
var (
- _ Store = (*badgerDB)(nil)
- _ IndexStore = (*badgerDB)(nil)
- _ y.Iterator = (*mergedIter)(nil)
- _ TimeSeriesStore = (*badgerTSS)(nil)
- bitMergeEntry byte = 1 << 3
- ErrKeyNotFound = badger.ErrKeyNotFound
+ _ Store = (*badgerDB)(nil)
+ _ IndexStore = (*badgerDB)(nil)
+ _ y.Iterator = (*mergedIter)(nil)
+ _ TimeSeriesStore = (*badgerTSS)(nil)
+ bitMergeEntry byte = 1 << 3
+ // ErrKeyNotFound denotes the expected key can not be got from the kv service.
+ ErrKeyNotFound = badger.ErrKeyNotFound
)
type badgerTSS struct {
@@ -147,12 +148,6 @@ func (b *badgerDB) Stats() observability.Statistics {
return badgerStats(b.db)
}
-func (b *badgerDB) Handover(iterator Iterator) error {
- return b.db.HandoverIterator(&mergedIter{
- delegated: iterator,
- })
-}
-
func (b *badgerDB) Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchSize = opt.PrefetchSize
@@ -173,7 +168,7 @@ func (b *badgerDB) Scan(prefix, seekKey []byte, opt ScanOpts, f ScanFunc) error
err := f(b.shardID, k, func() ([]byte, error) {
return y.Copy(it.Value().Value), nil
})
- if errors.Is(err, ErrStopScan) {
+ if errors.Is(err, errStopScan) {
break
}
if err != nil {
diff --git a/banyand/kv/kv.go b/banyand/kv/kv.go
index f180d55..fb3960a 100644
--- a/banyand/kv/kv.go
+++ b/banyand/kv/kv.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package kv implements a key-value engine.
package kv
import (
@@ -31,21 +32,25 @@ import (
)
var (
- ErrStopScan = errors.New("stop scanning")
+ errStopScan = errors.New("stop scanning")
+
+ // DefaultScanOpts is a helper to provides canonical options for scanning.
DefaultScanOpts = ScanOpts{
PrefetchSize: 100,
PrefetchValues: true,
}
)
-type Writer interface {
+type writer interface {
// Put a value
Put(key, val []byte) error
PutWithVersion(key, val []byte, version uint64) error
}
+// ScanFunc is the closure executed on scanning out a pair of key-value.
type ScanFunc func(shardID int, key []byte, getVal func() ([]byte, error)) error
+// ScanOpts wraps options for scanning the kv storage.
type ScanOpts struct {
Prefix []byte
PrefetchSize int
@@ -53,6 +58,7 @@ type ScanOpts struct {
Reverse bool
}
+// Reader allows retrieving data from kv.
type Reader interface {
Iterable
// Get a value by its key
@@ -65,10 +71,11 @@ type Reader interface {
type Store interface {
observability.Observable
io.Closer
- Writer
+ writer
Reader
}
+// TimeSeriesWriter allows writing to a time-series storage.
type TimeSeriesWriter interface {
// Put a value with a timestamp/version
Put(key, val []byte, ts uint64) error
@@ -77,6 +84,7 @@ type TimeSeriesWriter interface {
PutAsync(key, val []byte, ts uint64, f func(error)) error
}
+// TimeSeriesReader allows retrieving data from a time-series storage.
type TimeSeriesReader interface {
// Get a value by its key and timestamp/version
Get(key []byte, ts uint64) ([]byte, error)
@@ -91,6 +99,7 @@ type TimeSeriesStore interface {
TimeSeriesReader
}
+// TimeSeriesOptions sets an options for creating a TimeSeriesStore.
type TimeSeriesOptions func(TimeSeriesStore)
// TSSWithLogger sets a external logger into underlying TimeSeriesStore.
@@ -104,6 +113,7 @@ func TSSWithLogger(l *logger.Logger) TimeSeriesOptions {
}
}
+// TSSWithEncoding sets encoding and decoding pools for flushing and compacting.
func TSSWithEncoding(encoderPool encoding.SeriesEncoderPool, decoderPool encoding.SeriesDecoderPool) TimeSeriesOptions {
return func(store TimeSeriesStore) {
if btss, ok := store.(*badgerTSS); ok {
@@ -117,25 +127,20 @@ func TSSWithEncoding(encoderPool encoding.SeriesEncoderPool, decoderPool encodin
}
}
-func TSSWithFlushCallback(callback func()) TimeSeriesOptions {
+// TSSWithMemTableSize sets the size of memory table in bytes.
+func TSSWithMemTableSize(sizeInBytes int64) TimeSeriesOptions {
return func(store TimeSeriesStore) {
- if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts.FlushCallBack = callback
- }
- }
-}
-
-func TSSWithMemTableSize(size int64) TimeSeriesOptions {
- return func(store TimeSeriesStore) {
- if size < 1 {
+ if sizeInBytes < 1 {
return
}
if btss, ok := store.(*badgerTSS); ok {
- btss.dbOpts.MemTableSize = size
+ btss.dbOpts.MemTableSize = sizeInBytes
}
}
}
+// Iterator allows iterating the kv tables.
+// TODO: use generic to provide a unique iterator
type Iterator interface {
Next()
Rewind()
@@ -147,17 +152,16 @@ type Iterator interface {
Close() error
}
+// Iterable allows creating a Iterator.
type Iterable interface {
NewIterator(opt ScanOpts) Iterator
}
-type HandoverCallback func()
-
+// IndexStore allows writing and reading index format data.
type IndexStore interface {
observability.Observable
Iterable
Reader
- Handover(iterator Iterator) error
Close() error
}
@@ -184,6 +188,7 @@ func OpenTimeSeriesStore(shardID int, path string, options ...TimeSeriesOptions)
return btss, nil
}
+// StoreOptions sets options for creating Store.
type StoreOptions func(Store)
// StoreWithLogger sets a external logger into underlying Store.
@@ -236,6 +241,7 @@ func OpenStore(shardID int, path string, options ...StoreOptions) (Store, error)
return bdb, nil
}
+// IndexOptions sets options for creating the index store.
type IndexOptions func(store IndexStore)
// IndexWithLogger sets a external logger into underlying IndexStore.
diff --git a/banyand/liaison/grpc/discovery.go b/banyand/liaison/grpc/discovery.go
index cbff00e..bb9f66e 100644
--- a/banyand/liaison/grpc/discovery.go
+++ b/banyand/liaison/grpc/discovery.go
@@ -33,7 +33,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/partition"
)
-var ErrNotExist = errors.New("the object doesn't exist")
+var errNotExist = errors.New("the object doesn't exist")
type discoveryService struct {
shardRepo *shardRepo
@@ -61,11 +61,11 @@ func (ds *discoveryService) navigate(metadata *commonv1.Metadata, tagFamilies []
Name: metadata.Group,
}))
if !existed {
- return nil, nil, common.ShardID(0), errors.Wrapf(ErrNotExist, "finding the shard num by: %v", metadata)
+ return nil, nil, common.ShardID(0), errors.Wrapf(errNotExist, "finding the shard num by: %v", metadata)
}
locator, existed := ds.entityRepo.getLocator(getID(metadata))
if !existed {
- return nil, nil, common.ShardID(0), errors.Wrapf(ErrNotExist, "finding the locator by: %v", metadata)
+ return nil, nil, common.ShardID(0), errors.Wrapf(errNotExist, "finding the locator by: %v", metadata)
}
return locator.Locate(metadata.Name, tagFamilies, shardNum)
}
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index be38a38..a0c8d0b 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -43,10 +43,7 @@ type measureService struct {
func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) error {
reply := func() error {
- if err := measure.Send(&measurev1.WriteResponse{}); err != nil {
- return err
- }
- return nil
+ return measure.Send(&measurev1.WriteResponse{})
}
sampled := ms.log.Sample(&zerolog.BasicSampler{N: 10})
for {
@@ -118,7 +115,7 @@ func (ms *measureService) Query(_ context.Context, req *measurev1.QueryRequest)
case []*measurev1.DataPoint:
return &measurev1.QueryResponse{DataPoints: d}, nil
case common.Error:
- return nil, errors.WithMessage(ErrQueryMsg, d.Msg())
+ return nil, errors.WithMessage(errQueryMsg, d.Msg())
}
return nil, nil
}
@@ -142,7 +139,7 @@ func (ms *measureService) TopN(_ context.Context, topNRequest *measurev1.TopNReq
case []*measurev1.TopNList:
return &measurev1.TopNResponse{Lists: d}, nil
case common.Error:
- return nil, errors.WithMessage(ErrQueryMsg, d.Msg())
+ return nil, errors.WithMessage(errQueryMsg, d.Msg())
}
return nil, nil
}
diff --git a/banyand/liaison/grpc/registry.go b/banyand/liaison/grpc/registry.go
index 767cb6b..87927e7 100644
--- a/banyand/liaison/grpc/registry.go
+++ b/banyand/liaison/grpc/registry.go
@@ -400,7 +400,7 @@ func (rs *groupRegistryServer) Get(ctx context.Context, req *databasev1.GroupReg
}, nil
}
-func (rs *groupRegistryServer) List(ctx context.Context, req *databasev1.GroupRegistryServiceListRequest) (
+func (rs *groupRegistryServer) List(ctx context.Context, _ *databasev1.GroupRegistryServiceListRequest) (
*databasev1.GroupRegistryServiceListResponse, error,
) {
groups, err := rs.schemaRegistry.GroupRegistry().ListGroup(ctx)
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index 4aa66ba..5a58d80 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package grpc implements the gRPC services defined by APIs.
package grpc
import (
@@ -45,13 +46,13 @@ import (
const defaultRecvSize = 1024 * 1024 * 10
var (
- ErrServerCert = errors.New("invalid server cert file")
- ErrServerKey = errors.New("invalid server key file")
- ErrNoAddr = errors.New("no address")
- ErrQueryMsg = errors.New("invalid query message")
+ errServerCert = errors.New("invalid server cert file")
+ errServerKey = errors.New("invalid server key file")
+ errNoAddr = errors.New("no address")
+ errQueryMsg = errors.New("invalid query message")
)
-type Server struct {
+type server struct {
pipeline queue.Queue
creds credentials.TransportCredentials
repo discovery.ServiceRepo
@@ -74,8 +75,9 @@ type Server struct {
tls bool
}
-func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRepo, schemaRegistry metadata.Service) *Server {
- return &Server{
+// NewServer returns a new gRPC server.
+func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRepo, schemaRegistry metadata.Service) run.Unit {
+ return &server{
pipeline: pipeline,
repo: repo,
streamSVC: &streamService{
@@ -108,7 +110,7 @@ func NewServer(_ context.Context, pipeline queue.Queue, repo discovery.ServiceRe
}
}
-func (s *Server) PreRun() error {
+func (s *server) PreRun() error {
s.log = logger.GetLogger("liaison-grpc")
components := []struct {
discoverySVC *discoveryService
@@ -140,11 +142,11 @@ func (s *Server) PreRun() error {
return nil
}
-func (s *Server) Name() string {
+func (s *server) Name() string {
return "grpc"
}
-func (s *Server) FlagSet() *run.FlagSet {
+func (s *server) FlagSet() *run.FlagSet {
fs := run.NewFlagSet("grpc")
fs.IntVarP(&s.maxRecvMsgSize, "max-recv-msg-size", "", defaultRecvSize, "the size of max receiving message")
fs.BoolVarP(&s.tls, "tls", "", false, "connection uses TLS if true, else plain TCP")
@@ -154,18 +156,18 @@ func (s *Server) FlagSet() *run.FlagSet {
return fs
}
-func (s *Server) Validate() error {
+func (s *server) Validate() error {
if s.addr == "" {
- return ErrNoAddr
+ return errNoAddr
}
if !s.tls {
return nil
}
if s.certFile == "" {
- return ErrServerCert
+ return errServerCert
}
if s.keyFile == "" {
- return ErrServerKey
+ return errServerKey
}
creds, errTLS := credentials.NewServerTLSFromFile(s.certFile, s.keyFile)
if errTLS != nil {
@@ -175,7 +177,7 @@ func (s *Server) Validate() error {
return nil
}
-func (s *Server) Serve() run.StopNotify {
+func (s *server) Serve() run.StopNotify {
var opts []grpclib.ServerOption
if s.tls {
opts = []grpclib.ServerOption{grpclib.Creds(s.creds)}
@@ -215,7 +217,7 @@ func (s *Server) Serve() run.StopNotify {
return s.stopCh
}
-func (s *Server) GracefulStop() {
+func (s *server) GracefulStop() {
s.log.Info().Msg("stopping")
stopped := make(chan struct{})
go func() {
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 191df78..f70e188 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -43,10 +43,7 @@ type streamService struct {
func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
reply := func() error {
- if err := stream.Send(&streamv1.WriteResponse{}); err != nil {
- return err
- }
- return nil
+ return stream.Send(&streamv1.WriteResponse{})
}
sampled := s.log.Sample(&zerolog.BasicSampler{N: 10})
for {
@@ -127,7 +124,7 @@ func (s *streamService) Query(_ context.Context, req *streamv1.QueryRequest) (*s
case []*streamv1.Element:
return &streamv1.QueryResponse{Elements: d}, nil
case common.Error:
- return nil, errors.WithMessage(ErrQueryMsg, d.Msg())
+ return nil, errors.WithMessage(errQueryMsg, d.Msg())
}
return nil, nil
}
diff --git a/banyand/liaison/http/health.go b/banyand/liaison/http/health.go
index 7681e21..51f6272 100644
--- a/banyand/liaison/http/health.go
+++ b/banyand/liaison/http/health.go
@@ -56,7 +56,7 @@ type healthCheckClient struct {
conn *grpc.ClientConn
}
-func (g *healthCheckClient) Check(ctx context.Context, r *grpc_health_v1.HealthCheckRequest, opts ...grpc.CallOption) (*grpc_health_v1.HealthCheckResponse, error) {
+func (g *healthCheckClient) Check(ctx context.Context, _ *grpc_health_v1.HealthCheckRequest, _ ...grpc.CallOption) (*grpc_health_v1.HealthCheckResponse, error) {
var resp *grpc_health_v1.HealthCheckResponse
if err := grpchelper.Request(ctx, 10*time.Second, func(rpcCtx context.Context) (err error) {
resp, err = grpc_health_v1.NewHealthClient(g.conn).Check(rpcCtx,
@@ -70,6 +70,6 @@ func (g *healthCheckClient) Check(ctx context.Context, r *grpc_health_v1.HealthC
return resp, nil
}
-func (g *healthCheckClient) Watch(ctx context.Context, r *grpc_health_v1.HealthCheckRequest, opts ...grpc.CallOption) (grpc_health_v1.Health_WatchClient, error) {
+func (g *healthCheckClient) Watch(_ context.Context, _ *grpc_health_v1.HealthCheckRequest, _ ...grpc.CallOption) (grpc_health_v1.Health_WatchClient, error) {
return nil, status.Error(codes.Unimplemented, "unimplemented")
}
diff --git a/banyand/liaison/http/server.go b/banyand/liaison/http/server.go
index 4f51e25..6008bc9 100644
--- a/banyand/liaison/http/server.go
+++ b/banyand/liaison/http/server.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package http implements the gRPC gateway.
package http
import (
@@ -40,14 +41,13 @@ import (
"github.com/apache/skywalking-banyandb/ui"
)
-type ServiceRepo interface {
- run.Config
- run.Service
-}
-
-var _ ServiceRepo = (*service)(nil)
+var (
+ _ run.Config = (*service)(nil)
+ _ run.Service = (*service)(nil)
+)
-func NewService() ServiceRepo {
+// NewService return a http service.
+func NewService() run.Unit {
return &service{
stopCh: make(chan struct{}),
}
diff --git a/banyand/liaison/liaison.go b/banyand/liaison/liaison.go
index e5a19dc..bb59f22 100644
--- a/banyand/liaison/liaison.go
+++ b/banyand/liaison/liaison.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package liaison implements a transmission layer between a data layer and a client.
package liaison
import (
@@ -27,12 +28,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-type Endpoint interface {
- run.Config
- run.PreRunner
- run.Service
-}
-
-func NewEndpoint(ctx context.Context, pipeline queue.Queue, repo discovery.ServiceRepo, schemaRegistry metadata.Service) (Endpoint, error) {
+// NewEndpoint return a new endpoint which is the entry point for the database server.
+func NewEndpoint(ctx context.Context, pipeline queue.Queue, repo discovery.ServiceRepo, schemaRegistry metadata.Service) (run.Unit, error) {
return grpc.NewServer(ctx, pipeline, repo, schemaRegistry), nil
}
diff --git a/banyand/measure/field_flag_test.go b/banyand/measure/field_flag_test.go
index ae0845f..c47e457 100644
--- a/banyand/measure/field_flag_test.go
+++ b/banyand/measure/field_flag_test.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package measure
import (
diff --git a/banyand/measure/measure.go b/banyand/measure/measure.go
index 0cc528f..4a2e3be 100644
--- a/banyand/measure/measure.go
+++ b/banyand/measure/measure.go
@@ -15,14 +15,15 @@
// specific language governing permissions and limitations
// under the License.
+// Package measures implements a time-series-based storage which is consists of a sequence of data points.
+// Each data point contains tags and fields. They arrive in a fixed interval. A data point could be updated
+// by one with the identical entity(series_id) and timestamp.
package measure
import (
"context"
"time"
- "go.uber.org/multierr"
-
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
"github.com/apache/skywalking-banyandb/banyand/tsdb"
@@ -74,7 +75,7 @@ func (s *measure) EntityLocator() partition.EntityLocator {
}
func (s *measure) Close() error {
- return multierr.Combine(s.processorManager.Close(), s.indexWriter.Close())
+ return s.processorManager.Close()
}
func (s *measure) parseSpec() (err error) {
diff --git a/banyand/measure/measure_query.go b/banyand/measure/measure_query.go
index 1220e73..17ea882 100644
--- a/banyand/measure/measure_query.go
+++ b/banyand/measure/measure_query.go
@@ -35,13 +35,15 @@ import (
resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
)
-var ErrTagFamilyNotExist = errors.New("tag family doesn't exist")
+var errTagFamilyNotExist = errors.New("tag family doesn't exist")
+// Query allow to retrieve measure data points.
type Query interface {
LoadGroup(name string) (resourceSchema.Group, bool)
Measure(measure *commonv1.Metadata) (Measure, error)
}
+// Measure allows inspecting measure data points' details.
type Measure interface {
io.Closer
Shards(entity tsdb.Entity) ([]tsdb.Shard, error)
@@ -135,7 +137,7 @@ func (s *measure) ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFam
}
}
if tagSpec == nil {
- return nil, ErrTagFamilyNotExist
+ return nil, errTagFamilyNotExist
}
for i, tag := range tagFamily.GetTags() {
tags[i] = &modelv1.Tag{
diff --git a/banyand/measure/measure_topn.go b/banyand/measure/measure_topn.go
index e5ad23d..4d51099 100644
--- a/banyand/measure/measure_topn.go
+++ b/banyand/measure/measure_topn.go
@@ -47,7 +47,8 @@ import (
const (
timeBucketFormat = "200601021504"
- TopNTagFamily = "__topN__"
+ // TopNTagFamily is the identity of a tag family which contains the topN calculated result.
+ TopNTagFamily = "__topN__"
)
var (
@@ -57,6 +58,7 @@ var (
errUnsupportedConditionValueType = errors.New("unsupported value type in the condition")
+ // TopNValueFieldSpec denotes the field specification of the topN calculated result.
TopNValueFieldSpec = &databasev1.FieldSpec{
Name: "value",
FieldType: databasev1.FieldType_FIELD_TYPE_INT,
@@ -109,7 +111,7 @@ func (t *topNStreamingProcessor) run(ctx context.Context) {
// Teardown is called by the Flow as a lifecycle hook.
// So we should not block on err channel within this method.
-func (t *topNStreamingProcessor) Teardown(ctx context.Context) error {
+func (t *topNStreamingProcessor) Teardown(_ context.Context) error {
t.Wait()
return nil
}
@@ -280,13 +282,13 @@ func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
streaming.WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
return record.Data().(flow.Data)[1].(int64)
}),
- OrderBy(t.topNSchema.GetFieldValueSort()),
+ orderBy(t.topNSchema.GetFieldValueSort()),
).To(t).Open()
go t.handleError()
return t
}
-func OrderBy(sort modelv1.Sort) streaming.TopNOption {
+func orderBy(sort modelv1.Sort) streaming.TopNOption {
if sort == modelv1.Sort_SORT_ASC {
return streaming.OrderBy(streaming.ASC)
}
diff --git a/banyand/measure/measure_write.go b/banyand/measure/measure_write.go
index ff26fc2..a29c535 100644
--- a/banyand/measure/measure_write.go
+++ b/banyand/measure/measure_write.go
@@ -37,7 +37,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-var ErrMalformedElement = errors.New("element is malformed")
+var errMalformedElement = errors.New("element is malformed")
func (s *measure) write(md *commonv1.Metadata, shardID common.ShardID, entity []byte, entityValues tsdb.EntityValues, value *measurev1.DataPointValue) error {
t := value.GetTimestamp().AsTime().Local()
@@ -47,10 +47,10 @@ func (s *measure) write(md *commonv1.Metadata, shardID common.ShardID, entity []
sm := s.schema
fLen := len(value.GetTagFamilies())
if fLen < 1 {
- return errors.Wrap(ErrMalformedElement, "no tag family")
+ return errors.Wrap(errMalformedElement, "no tag family")
}
if fLen > len(sm.TagFamilies) {
- return errors.Wrap(ErrMalformedElement, "tag family number is more than expected")
+ return errors.Wrap(errMalformedElement, "tag family number is more than expected")
}
shard, err := s.databaseSupplier.SupplyTSDB().Shard(shardID)
if err != nil {
@@ -80,7 +80,7 @@ func (s *measure) write(md *commonv1.Metadata, shardID common.ShardID, entity []
builder.Family(familyIdentity(spec.GetName(), pbv1.TagFlag), bb)
}
if len(value.GetFields()) > len(sm.GetFields()) {
- return nil, errors.Wrap(ErrMalformedElement, "fields number is more than expected")
+ return nil, errors.Wrap(errMalformedElement, "fields number is more than expected")
}
for fi, fieldValue := range value.GetFields() {
fieldSpec := sm.GetFields()[fi]
@@ -90,7 +90,7 @@ func (s *measure) write(md *commonv1.Metadata, shardID common.ShardID, entity []
continue
}
if fType != fieldSpec.GetFieldType() {
- return nil, errors.Wrapf(ErrMalformedElement, "field %s type is unexpected", fieldSpec.GetName())
+ return nil, errors.Wrapf(errMalformedElement, "field %s type is unexpected", fieldSpec.GetName())
}
data := encodeFieldValue(fieldValue)
if data == nil {
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index 88aa56c..06b1f97 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -209,7 +209,7 @@ func (s *supplier) OpenResource(shardNum uint32, db tsdb.Supplier, spec resource
}, s.l)
}
-func (s *supplier) ResourceSchema(repo metadata.Repo, md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
+func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return s.metadata.MeasureRegistry().GetMeasure(ctx, md)
diff --git a/banyand/measure/service.go b/banyand/measure/service.go
index 6b15ede..60e615d 100644
--- a/banyand/measure/service.go
+++ b/banyand/measure/service.go
@@ -38,10 +38,12 @@ import (
)
var (
- ErrEmptyRootPath = errors.New("root path is empty")
+ errEmptyRootPath = errors.New("root path is empty")
+ // ErrMeasureNotExist denotes a measure doesn't exist in the metadata repo.
ErrMeasureNotExist = errors.New("measure doesn't exist")
)
+// Service allows inspecting the measure data points.
type Service interface {
run.PreRunner
run.Config
@@ -85,7 +87,7 @@ func (s *service) FlagSet() *run.FlagSet {
func (s *service) Validate() error {
if s.root == "" {
- return ErrEmptyRootPath
+ return errEmptyRootPath
}
return nil
}
diff --git a/banyand/metadata/metadata.go b/banyand/metadata/metadata.go
index acaf677..44f5953 100644
--- a/banyand/metadata/metadata.go
+++ b/banyand/metadata/metadata.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// Package metadata implements a Raft-based distributed metadata storage system.
+// Powered by etcd.
package metadata
import (
@@ -39,6 +41,7 @@ type IndexFilter interface {
Subjects(ctx context.Context, indexRule *databasev1.IndexRule, catalog commonv1.Catalog) ([]schema.Spec, error)
}
+// Repo is the facade to interact with the metadata repository.
type Repo interface {
IndexFilter
StreamRegistry() schema.Stream
@@ -50,6 +53,7 @@ type Repo interface {
PropertyRegistry() schema.Property
}
+// Service is the metadata repository.
type Service interface {
Repo
run.PreRunner
@@ -101,6 +105,7 @@ func (s *service) GracefulStop() {
<-s.schemaRegistry.StopNotify()
}
+// NewService returns a new metadata repository Service.
func NewService(_ context.Context) (Service, error) {
return &service{}, nil
}
diff --git a/banyand/observability/metric.go b/banyand/observability/metric.go
index b4598c6..9430237 100644
--- a/banyand/observability/metric.go
+++ b/banyand/observability/metric.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package observability
import (
@@ -31,6 +32,7 @@ var (
_ run.Config = (*metricService)(nil)
)
+// NewMetricService returns a metric service.
func NewMetricService() run.Service {
return &metricService{
closer: run.NewCloser(1),
diff --git a/banyand/observability/pprof.go b/banyand/observability/pprof.go
index 1340f9b..0e65136 100644
--- a/banyand/observability/pprof.go
+++ b/banyand/observability/pprof.go
@@ -31,6 +31,7 @@ var (
_ run.Config = (*metricService)(nil)
)
+// NewProfService returns a pprof service.
func NewProfService() run.Service {
return &pprofService{
closer: run.NewCloser(1),
diff --git a/banyand/observability/type.go b/banyand/observability/type.go
index f6829d1..d6e4b09 100644
--- a/banyand/observability/type.go
+++ b/banyand/observability/type.go
@@ -14,17 +14,21 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
+// Package observability provides metrics, profiling, and etc.
package observability
import "errors"
var errNoAddr = errors.New("no address")
+// Statistics represents a sample of a module.
type Statistics struct {
MemBytes int64
MaxMemBytes int64
}
+// Observable allows sampling.
type Observable interface {
Stats() Statistics
}
diff --git a/banyand/query/processor.go b/banyand/query/processor.go
index 43100f2..6171b1a 100644
--- a/banyand/query/processor.go
+++ b/banyand/query/processor.go
@@ -37,6 +37,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/query/executor"
logical_measure "github.com/apache/skywalking-banyandb/pkg/query/logical/measure"
logical_stream "github.com/apache/skywalking-banyandb/pkg/query/logical/stream"
+ "github.com/apache/skywalking-banyandb/pkg/run"
)
const (
@@ -44,7 +45,7 @@ const (
)
var (
- _ Executor = (*queryService)(nil)
+ _ run.PreRunner = (*queryService)(nil)
_ bus.MessageListener = (*streamQueryProcessor)(nil)
_ bus.MessageListener = (*measureQueryProcessor)(nil)
_ bus.MessageListener = (*topNQueryProcessor)(nil)
diff --git a/banyand/query/query.go b/banyand/query/query.go
index 267a922..3e4eab1 100644
--- a/banyand/query/query.go
+++ b/banyand/query/query.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package query implement the query module for liaison and other modules to retrieve data.
package query
import (
@@ -28,13 +29,10 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
-type Executor interface {
- run.PreRunner
-}
-
-func NewExecutor(_ context.Context, streamService stream.Service, measureService measure.Service,
+// NewService return a new query service.
+func NewService(_ context.Context, streamService stream.Service, measureService measure.Service,
metaService metadata.Service, serviceRepo discovery.ServiceRepo, pipeline queue.Queue,
-) (Executor, error) {
+) (run.Unit, error) {
svc := &queryService{
metaService: metaService,
serviceRepo: serviceRepo,
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index 1061cbf..ec4a95e 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-// Package queue implements the data transmission queue
+// Package queue implements the data transmission queue.
package queue
import (
diff --git a/banyand/queue/queue.go b/banyand/queue/queue.go
index 7941a9b..0946ecb 100644
--- a/banyand/queue/queue.go
+++ b/banyand/queue/queue.go
@@ -25,6 +25,8 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
+// Queue builds a data transmission tunnel between subscribers and publishers.
+//
//go:generate mockgen -destination=./queue_mock.go -package=queue github.com/apache/skywalking-banyandb/pkg/bus MessageListener
type Queue interface {
run.Unit
@@ -33,6 +35,7 @@ type Queue interface {
run.Service
}
+// NewQueue return a new Queue which relies on the discovery service.
func NewQueue(_ context.Context, repo discovery.ServiceRepo) (Queue, error) {
return &local{
repo: repo,
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 5b200d9..286a276 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -195,7 +195,7 @@ func (s *supplier) OpenResource(shardNum uint32, db tsdb.Supplier, spec resource
}, s.l), nil
}
-func (s *supplier) ResourceSchema(repo metadata.Repo, md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
+func (s *supplier) ResourceSchema(md *commonv1.Metadata) (resourceSchema.ResourceSchema, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return s.metadata.StreamRegistry().GetStream(ctx, md)
diff --git a/banyand/stream/service.go b/banyand/stream/service.go
index 77c4989..ac95748 100644
--- a/banyand/stream/service.go
+++ b/banyand/stream/service.go
@@ -36,10 +36,11 @@ import (
)
var (
- ErrEmptyRootPath = errors.New("root path is empty")
- ErrStreamNotExist = errors.New("stream doesn't exist")
+ errEmptyRootPath = errors.New("root path is empty")
+ errStreamNotExist = errors.New("stream doesn't exist")
)
+// Service allows inspecting the stream elements.
type Service interface {
run.PreRunner
run.Config
@@ -64,7 +65,7 @@ type service struct {
func (s *service) Stream(metadata *commonv1.Metadata) (Stream, error) {
sm, ok := s.schemaRepo.loadStream(metadata)
if !ok {
- return nil, errors.WithStack(ErrStreamNotExist)
+ return nil, errors.WithStack(errStreamNotExist)
}
return sm, nil
}
@@ -80,7 +81,7 @@ func (s *service) FlagSet() *run.FlagSet {
func (s *service) Validate() error {
if s.root == "" {
- return ErrEmptyRootPath
+ return errEmptyRootPath
}
return nil
}
diff --git a/banyand/stream/stream.go b/banyand/stream/stream.go
index 41f45a4..c8866cf 100644
--- a/banyand/stream/stream.go
+++ b/banyand/stream/stream.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// Package stream implements a time-series-based storage which is consists of a sequence of element.
+// Each element drops in a arbitrary interval. They are immutable, can not be updated or overwritten.
package stream
import (
@@ -65,7 +67,7 @@ func (s *stream) EntityLocator() partition.EntityLocator {
}
func (s *stream) Close() error {
- return s.indexWriter.Close()
+ return nil
}
func (s *stream) parseSpec() {
diff --git a/banyand/stream/stream_query.go b/banyand/stream/stream_query.go
index 1f89be8..5b3ef7c 100644
--- a/banyand/stream/stream_query.go
+++ b/banyand/stream/stream_query.go
@@ -31,12 +31,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/partition"
)
-var ErrTagFamilyNotExist = errors.New("tag family doesn't exist")
+var errTagFamilyNotExist = errors.New("tag family doesn't exist")
+// Query allow to retrieve elements in a series of streams.
type Query interface {
Stream(stream *commonv1.Metadata) (Stream, error)
}
+// Stream allows inspecting elements' details.
type Stream interface {
io.Closer
Shards(entity tsdb.Entity) ([]tsdb.Shard, error)
@@ -101,7 +103,7 @@ func (s *stream) ParseTagFamily(family string, item tsdb.Item) (*modelv1.TagFami
}
}
if tagSpec == nil {
- return nil, ErrTagFamilyNotExist
+ return nil, errTagFamilyNotExist
}
for i, tag := range tagFamily.GetTags() {
tags[i] = &modelv1.Tag{
diff --git a/banyand/stream/stream_write.go b/banyand/stream/stream_write.go
index 3a77da3..01bb656 100644
--- a/banyand/stream/stream_write.go
+++ b/banyand/stream/stream_write.go
@@ -36,7 +36,7 @@ import (
)
var (
- ErrMalformedElement = errors.New("element is malformed")
+ errMalformedElement = errors.New("element is malformed")
writtenBytes *prometheus.CounterVec
)
@@ -59,10 +59,10 @@ func (s *stream) write(shardID common.ShardID, entity []byte, entityValues tsdb.
sm := s.schema
fLen := len(value.GetTagFamilies())
if fLen < 1 {
- return errors.Wrap(ErrMalformedElement, "no tag family")
+ return errors.Wrap(errMalformedElement, "no tag family")
}
if fLen > len(sm.TagFamilies) {
- return errors.Wrap(ErrMalformedElement, "tag family number is more than expected")
+ return errors.Wrap(errMalformedElement, "tag family number is more than expected")
}
shard, err := s.db.SupplyTSDB().Shard(shardID)
if err != nil {
diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go
index d5c48cf..cdfd8f2 100644
--- a/banyand/tsdb/block.go
+++ b/banyand/tsdb/block.go
@@ -53,7 +53,7 @@ const (
defaultEnqueueTimeout = 500 * time.Millisecond
)
-var ErrBlockClosingInterrupted = errors.New("interrupt to close the block")
+var errBlockClosingInterrupted = errors.New("interrupt to close the block")
type block struct {
encodingMethod EncodingMethod
@@ -189,9 +189,9 @@ func (b *block) open() (err error) {
return nil
}
-func (b *block) delegate(ctx context.Context) (BlockDelegate, error) {
+func (b *block) delegate(ctx context.Context) (blockDelegate, error) {
if b.deleted.Load() {
- return nil, errors.WithMessagef(ErrBlockAbsent, "block %s is deleted", b)
+ return nil, errors.WithMessagef(errBlockAbsent, "block %s is deleted", b)
}
blockID := BlockID{
BlockID: b.blockID,
@@ -276,7 +276,7 @@ func (b *block) close(ctx context.Context) (err error) {
select {
case <-ctx.Done():
stopWaiting.Store(true)
- return errors.Wrapf(ErrBlockClosingInterrupted, "block:%s", b)
+ return errors.Wrapf(errBlockClosingInterrupted, "block:%s", b)
case <-ch:
}
b.closed.Store(true)
@@ -313,7 +313,7 @@ func (b *block) stats() (names []string, stats []observability.Statistics) {
return names, stats
}
-type BlockDelegate interface {
+type blockDelegate interface {
io.Closer
contains(ts time.Time) bool
write(key []byte, val []byte, ts time.Time) error
@@ -330,7 +330,7 @@ type BlockDelegate interface {
String() string
}
-var _ BlockDelegate = (*bDelegate)(nil)
+var _ blockDelegate = (*bDelegate)(nil)
type bDelegate struct {
delegate *block
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
index 475b218..b1071d3 100644
--- a/banyand/tsdb/block_ctrl.go
+++ b/banyand/tsdb/block_ctrl.go
@@ -93,7 +93,7 @@ func (bc *blockController) Next() (bucket.Reporter, error) {
}
b := c.(*block)
- return bc.newHeadBlock(bc.blockSize.NextTime(b.Start))
+ return bc.newHeadBlock(bc.blockSize.nextTime(b.Start))
}
func (bc *blockController) newHeadBlock(now time.Time) (*block, error) {
@@ -157,14 +157,14 @@ func (bc *blockController) Parse(value string) (time.Time, error) {
panic("invalid interval unit")
}
-func (bc *blockController) span(ctx context.Context, timeRange timestamp.TimeRange) ([]BlockDelegate, error) {
+func (bc *blockController) span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error) {
bb := bc.search(func(b *block) bool {
return b.Overlapping(timeRange)
})
if bb == nil {
return nil, nil
}
- dd := make([]BlockDelegate, len(bb))
+ dd := make([]blockDelegate, len(bb))
for i, b := range bb {
d, err := b.delegate(ctx)
if err != nil {
@@ -175,7 +175,7 @@ func (bc *blockController) span(ctx context.Context, timeRange timestamp.TimeRan
return dd, nil
}
-func (bc *blockController) get(ctx context.Context, blockID SectionID) (BlockDelegate, error) {
+func (bc *blockController) get(ctx context.Context, blockID SectionID) (blockDelegate, error) {
b := bc.getBlock(blockID)
if b != nil {
return b.delegate(ctx)
@@ -251,7 +251,7 @@ func (bc *blockController) create(start time.Time) (*block, error) {
next = s
}
}
- stdEnd := bc.blockSize.NextTime(start)
+ stdEnd := bc.blockSize.nextTime(start)
var end time.Time
if next != nil && next.Start.Before(stdEnd) {
end = next.Start
diff --git a/banyand/tsdb/bucket/bucket.go b/banyand/tsdb/bucket/bucket.go
index 6bdb01f..e05a400 100644
--- a/banyand/tsdb/bucket/bucket.go
+++ b/banyand/tsdb/bucket/bucket.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package bucket implements a rolling bucket system.
package bucket
import (
@@ -29,21 +30,25 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-var ErrReporterClosed = errors.New("reporter is closed")
+var errReporterClosed = errors.New("reporter is closed")
+// Controller defines the provider of a Reporter.
type Controller interface {
Current() (Reporter, error)
Next() (Reporter, error)
OnMove(prev, next Reporter)
}
+// Status is a sample of the Reporter's status.
type Status struct {
Capacity int
Volume int
}
+// Channel reports the status of a Reporter.
type Channel chan Status
+// Reporter allows reporting status to its supervisor.
type Reporter interface {
// TODO: refactor Report to return a status. It's too complicated to return a channel
Report() (Channel, error)
@@ -51,15 +56,16 @@ type Reporter interface {
}
var (
- _ Reporter = (*dummyReporter)(nil)
- _ Reporter = (*timeBasedReporter)(nil)
- DummyReporter = &dummyReporter{}
+ _ Reporter = (*dummyReporter)(nil)
+ _ Reporter = (*timeBasedReporter)(nil)
+ // DummyReporter is a special Reporter to avoid nil errors.
+ DummyReporter = &dummyReporter{}
)
type dummyReporter struct{}
func (*dummyReporter) Report() (Channel, error) {
- return nil, ErrReporterClosed
+ return nil, errReporterClosed
}
func (*dummyReporter) Stop() {
@@ -77,6 +83,7 @@ type timeBasedReporter struct {
name string
}
+// NewTimeBasedReporter returns a Reporter which sends report based on time.
func NewTimeBasedReporter(name string, timeRange timestamp.TimeRange, clock timestamp.Clock, scheduler *timestamp.Scheduler) Reporter {
if timeRange.End.Before(clock.Now()) {
return DummyReporter
@@ -93,11 +100,11 @@ func NewTimeBasedReporter(name string, timeRange timestamp.TimeRange, clock time
func (tr *timeBasedReporter) Report() (Channel, error) {
if tr.scheduler.Closed() {
- return nil, ErrReporterClosed
+ return nil, errReporterClosed
}
now := tr.clock.Now()
if now.After(tr.End) {
- return nil, ErrReporterClosed
+ return nil, errReporterClosed
}
ch := make(Channel, 1)
interval := tr.Duration() >> 4
@@ -137,7 +144,7 @@ func (tr *timeBasedReporter) Report() (Channel, error) {
}); err != nil {
close(ch)
if errors.Is(err, timestamp.ErrSchedulerClosed) {
- return nil, ErrReporterClosed
+ return nil, errReporterClosed
}
return nil, err
}
diff --git a/banyand/tsdb/bucket/queue.go b/banyand/tsdb/bucket/queue.go
index 18e900a..6cb1bfa 100644
--- a/banyand/tsdb/bucket/queue.go
+++ b/banyand/tsdb/bucket/queue.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package bucket
import (
@@ -31,10 +32,13 @@ import (
)
type (
- EvictFn func(ctx context.Context, id interface{}) error
+ // EvictFn is a closure executed on evicting an item.
+ EvictFn func(ctx context.Context, id interface{}) error
+ // OnAddRecentFn is a notifier on adding an item into the recent queue.
OnAddRecentFn func() error
)
+// Queue is a LRU queue.
type Queue interface {
Touch(id fmt.Stringer) bool
Push(ctx context.Context, id fmt.Stringer, fn OnAddRecentFn) error
@@ -45,13 +49,14 @@ type Queue interface {
}
const (
- QueueName = "block-queue-cleanup"
- DefaultRecentRatio = 0.25
+ // QueueName is identity of the queue.
+ QueueName = "block-queue-cleanup"
+ defaultRecentRatio = 0.25
defaultEvictBatchSize = 10
)
-var ErrInvalidSize = errors.New("invalid size")
+var errInvalidSize = errors.New("invalid size")
type lruQueue struct {
recent simplelru.LRUCache
@@ -65,12 +70,13 @@ type lruQueue struct {
lock sync.RWMutex
}
+// NewQueue return a Queue for blocks eviction.
func NewQueue(l *logger.Logger, size int, maxSize int, scheduler *timestamp.Scheduler, evictFn EvictFn) (Queue, error) {
if size <= 0 {
- return nil, ErrInvalidSize
+ return nil, errInvalidSize
}
- recentSize := int(float64(size) * DefaultRecentRatio)
+ recentSize := int(float64(size) * defaultRecentRatio)
evictSize := maxSize - size
recent, err := simplelru.NewLRU(size, nil)
diff --git a/banyand/tsdb/bucket/strategy.go b/banyand/tsdb/bucket/strategy.go
index dda86d8..31b601d 100644
--- a/banyand/tsdb/bucket/strategy.go
+++ b/banyand/tsdb/bucket/strategy.go
@@ -30,41 +30,48 @@ import (
)
var (
+ // ErrInvalidParameter denotes input parameters are invalid.
ErrInvalidParameter = errors.New("parameters are invalid")
- ErrNoMoreBucket = errors.New("no more buckets")
+ // ErrNoMoreBucket denotes the bucket volume reaches the limitation.
+ ErrNoMoreBucket = errors.New("no more buckets")
)
-type Ratio float64
+type ratio float64
+// Strategy controls Reporters with Controller's help.
type Strategy struct {
optionsErr error
ctrl Controller
current atomic.Value
logger *logger.Logger
closer *run.Closer
- ratio Ratio
+ ratio ratio
currentRatio uint64
}
+// StrategyOptions sets how to create a Strategy.
type StrategyOptions func(*Strategy)
-func WithNextThreshold(ratio Ratio) StrategyOptions {
+// WithNextThreshold sets a ratio to creat the next Reporter.
+func WithNextThreshold(r ratio) StrategyOptions {
return func(s *Strategy) {
- if ratio > 1.0 {
+ if r > 1.0 {
s.optionsErr = multierr.Append(s.optionsErr,
- errors.Wrapf(ErrInvalidParameter, "ratio %v is more than 1.0", ratio))
+ errors.Wrapf(ErrInvalidParameter, "ratio %v is more than 1.0", r))
return
}
- s.ratio = ratio
+ s.ratio = r
}
}
+// WithLogger sets a logger.Logger.
func WithLogger(logger *logger.Logger) StrategyOptions {
return func(s *Strategy) {
s.logger = logger
}
}
+// NewStrategy returns a Strategy.
func NewStrategy(ctrl Controller, options ...StrategyOptions) (*Strategy, error) {
if ctrl == nil {
return nil, errors.Wrap(ErrInvalidParameter, "controller is absent")
@@ -98,12 +105,13 @@ func (s *Strategy) resetCurrent() error {
return nil
}
+// Run the Strategy in the background.
func (s *Strategy) Run() {
go func(s *Strategy) {
defer s.closer.Done()
for {
c, err := s.current.Load().(Reporter).Report()
- if errors.Is(err, ErrReporterClosed) {
+ if errors.Is(err, errReporterClosed) {
return
}
if err != nil {
@@ -138,9 +146,9 @@ func (s *Strategy) observe(c Channel) bool {
if !more {
return moreBucket
}
- ratio := Ratio(status.Volume) / Ratio(status.Capacity)
- atomic.StoreUint64(&s.currentRatio, math.Float64bits(float64(ratio)))
- if ratio >= s.ratio && next == nil && moreBucket {
+ r := ratio(status.Volume) / ratio(status.Capacity)
+ atomic.StoreUint64(&s.currentRatio, math.Float64bits(float64(r)))
+ if r >= s.ratio && next == nil && moreBucket {
n, err := s.ctrl.Next()
if errors.Is(err, ErrNoMoreBucket) {
moreBucket = false
@@ -151,7 +159,7 @@ func (s *Strategy) observe(c Channel) bool {
next = n
}
}
- if ratio >= 1.0 {
+ if r >= 1.0 {
s.ctrl.OnMove(s.current.Load().(Reporter), next)
if next != nil {
s.current.Store(next)
@@ -164,6 +172,7 @@ func (s *Strategy) observe(c Channel) bool {
}
}
+// Close the Strategy running in the background.
func (s *Strategy) Close() {
s.closer.CloseThenWait()
}
diff --git a/banyand/tsdb/index/writer.go b/banyand/tsdb/index/writer.go
index 5e98d04..7385466 100644
--- a/banyand/tsdb/index/writer.go
+++ b/banyand/tsdb/index/writer.go
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+// Package index implements transferring data to indices.
package index
import (
@@ -35,8 +36,7 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
-type CallbackFn func()
-
+// Message wraps value and other info to generate relative indices.
type Message struct {
Value Value
LocalWriter tsdb.Writer
@@ -44,11 +44,13 @@ type Message struct {
Scope tsdb.Entry
}
+// Value represents the input data for generating indices.
type Value struct {
Timestamp time.Time
TagFamilies []*modelv1.TagFamilyForWrite
}
+// WriterOptions wrap all options to create an index writer.
type WriterOptions struct {
DB tsdb.Supplier
Families []*databasev1.TagFamilySpec
@@ -64,6 +66,7 @@ const (
tree
)
+// Writer generates indices based on index rules.
type Writer struct {
db tsdb.Supplier
l *logger.Logger
@@ -72,6 +75,7 @@ type Writer struct {
enableGlobalIndex bool
}
+// NewWriter returns a new Writer with WriterOptions.
func NewWriter(ctx context.Context, options WriterOptions) *Writer {
w := new(Writer)
parentLogger := ctx.Value(logger.ContextKey)
@@ -127,10 +131,6 @@ func (s *Writer) Write(m Message) {
}
}
-func (s *Writer) Close() error {
- return nil
-}
-
// TODO: should listen to pipeline in a distributed cluster.
func (s *Writer) writeGlobalIndex(scope tsdb.Entry, ref tsdb.GlobalItemID, value Value) error {
collect := func(ruleIndexes []*partition.IndexRuleLocator, fn func(indexWriter tsdb.IndexWriter, fields []index.Field) error) error {
@@ -225,12 +225,12 @@ func (s *Writer) writeLocalIndex(writer tsdb.Writer, value Value) (err error) {
)
}
-var ErrUnsupportedIndexType = errors.New("unsupported index type")
+var errUnsupportedIndexType = errors.New("unsupported index type")
func getIndexValue(ruleIndex *partition.IndexRuleLocator, value Value) (val [][]byte, err error) {
val = make([][]byte, 0)
if len(ruleIndex.TagIndices) != 1 {
- return nil, errors.WithMessagef(ErrUnsupportedIndexType,
+ return nil, errors.WithMessagef(errUnsupportedIndexType,
"the index rule %s(%v) didn't support composited tags",
ruleIndex.Rule.Metadata.Name, ruleIndex.Rule.Tags)
}
diff --git a/banyand/tsdb/indexdb.go b/banyand/tsdb/indexdb.go
index 797b111..cb82f88 100644
--- a/banyand/tsdb/indexdb.go
+++ b/banyand/tsdb/indexdb.go
@@ -30,16 +30,19 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index"
)
+// IndexDatabase allows stocking index data.
type IndexDatabase interface {
WriterBuilder() IndexWriterBuilder
Seek(field index.Field) ([]GlobalItemID, error)
}
+// IndexWriter allows ingesting index data.
type IndexWriter interface {
WriteLSMIndex(field []index.Field) error
WriteInvertedIndex(field []index.Field) error
}
+// IndexWriterBuilder is a helper to build IndexWriter.
type IndexWriterBuilder interface {
Scope(scope Entry) IndexWriterBuilder
Time(ts time.Time) IndexWriterBuilder
@@ -47,8 +50,6 @@ type IndexWriterBuilder interface {
Build() (IndexWriter, error)
}
-type IndexSeekBuilder interface{}
-
var _ IndexDatabase = (*indexDB)(nil)
type indexDB struct {
@@ -65,7 +66,7 @@ func (i *indexDB) Seek(field index.Field) ([]GlobalItemID, error) {
for _, s := range i.segCtrl.segments() {
err = s.globalIndex.GetAll(f, func(rawBytes []byte) error {
id := &GlobalItemID{}
- errUnMarshal := id.UnMarshal(rawBytes)
+ errUnMarshal := id.unMarshal(rawBytes)
if errUnMarshal != nil {
return errUnMarshal
}
@@ -120,7 +121,7 @@ func (i *indexWriterBuilder) Build() (IndexWriter, error) {
return nil, err
}
if i.globalItemID == nil {
- return nil, errors.WithStack(ErrNoVal)
+ return nil, errors.WithStack(errNoVal)
}
return &indexWriter{
scope: i.scope,
@@ -155,7 +156,7 @@ func (i *indexWriter) WriteLSMIndex(fields []index.Field) (err error) {
err = multierr.Append(err, errInternal)
continue
}
- err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(key, i.itemID.Marshal(), uint64(i.ts.UnixNano())))
+ err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(key, i.itemID.marshal(), uint64(i.ts.UnixNano())))
}
return err
}
@@ -170,11 +171,12 @@ func (i *indexWriter) WriteInvertedIndex(fields []index.Field) (err error) {
err = multierr.Append(err, errInternal)
continue
}
- err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(key, i.itemID.Marshal(), uint64(i.ts.UnixNano())))
+ err = multierr.Append(err, i.seg.globalIndex.PutWithVersion(key, i.itemID.marshal(), uint64(i.ts.UnixNano())))
}
return err
}
+// GlobalSeriesID encodes Entry to common.SeriesID.
func GlobalSeriesID(scope Entry) common.SeriesID {
return common.SeriesID(convert.Hash(scope))
}
diff --git a/banyand/tsdb/metric.go b/banyand/tsdb/metric.go
index cd99c68..40340b5 100644
--- a/banyand/tsdb/metric.go
+++ b/banyand/tsdb/metric.go
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package tsdb
import (
diff --git a/banyand/tsdb/retention.go b/banyand/tsdb/retention.go
index 6eb2e0f..43ff3f3 100644
--- a/banyand/tsdb/retention.go
+++ b/banyand/tsdb/retention.go
@@ -47,7 +47,7 @@ func newRetentionTask(segment *segmentController, ttl IntervalRule) *retentionTa
segment: segment,
option: cron.Minute | cron.Hour,
expr: expr,
- duration: ttl.EstimatedDuration(),
+ duration: ttl.estimatedDuration(),
}
}
diff --git a/banyand/tsdb/scope.go b/banyand/tsdb/scope.go
index ab9711c..952fd3c 100644
--- a/banyand/tsdb/scope.go
+++ b/banyand/tsdb/scope.go
@@ -24,45 +24,46 @@ import (
"github.com/apache/skywalking-banyandb/banyand/observability"
)
-var _ Shard = (*ScopedShard)(nil)
+var _ Shard = (*scopedShard)(nil)
-type ScopedShard struct {
+type scopedShard struct {
delegated Shard
scope Entry
}
+// NewScopedShard returns a shard in a scope.
func NewScopedShard(scope Entry, delegated Shard) Shard {
- return &ScopedShard{
+ return &scopedShard{
scope: scope,
delegated: delegated,
}
}
-func (sd *ScopedShard) Close() error {
+func (sd *scopedShard) Close() error {
// the delegate can't close the underlying shard
return nil
}
-func (sd *ScopedShard) ID() common.ShardID {
+func (sd *scopedShard) ID() common.ShardID {
return sd.delegated.ID()
}
-func (sd *ScopedShard) Series() SeriesDatabase {
+func (sd *scopedShard) Series() SeriesDatabase {
return &scopedSeriesDatabase{
scope: sd.scope,
delegated: sd.delegated.Series(),
}
}
-func (sd *ScopedShard) Index() IndexDatabase {
+func (sd *scopedShard) Index() IndexDatabase {
return sd.delegated.Index()
}
-func (sd *ScopedShard) TriggerSchedule(task string) bool {
+func (sd *scopedShard) TriggerSchedule(task string) bool {
return sd.delegated.TriggerSchedule(task)
}
-func (sd *ScopedShard) State() ShardState {
+func (sd *scopedShard) State() ShardState {
return sd.delegated.State()
}
@@ -90,5 +91,5 @@ func (sdd *scopedSeriesDatabase) GetByID(id common.SeriesID) (Series, error) {
}
func (sdd *scopedSeriesDatabase) List(ctx context.Context, path Path) (SeriesList, error) {
- return sdd.delegated.List(ctx, path.Prepend(sdd.scope))
+ return sdd.delegated.List(ctx, path.prepend(sdd.scope))
}
diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go
index 4a21a71..c6582f2 100644
--- a/banyand/tsdb/segment.go
+++ b/banyand/tsdb/segment.go
@@ -32,7 +32,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
-var ErrEndOfSegment = errors.New("reached the end of the segment")
+var errEndOfSegment = errors.New("reached the end of the segment")
type segment struct {
globalIndex kv.Store
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
index de02ef1..98653a2 100644
--- a/banyand/tsdb/segment_ctrl.go
+++ b/banyand/tsdb/segment_ctrl.go
@@ -134,8 +134,8 @@ func (sc *segmentController) Next() (bucket.Reporter, error) {
return nil, err
}
seg := c.(*segment)
- reporter, err := sc.create(sc.segmentSize.NextTime(seg.Start))
- if errors.Is(err, ErrEndOfSegment) {
+ reporter, err := sc.create(sc.segmentSize.nextTime(seg.Start))
+ if errors.Is(err, errEndOfSegment) {
return nil, bucket.ErrNoMoreBucket
}
return reporter, err
@@ -187,7 +187,7 @@ func (sc *segmentController) open() error {
defer sc.Unlock()
return loadSections(sc.location, sc, sc.segmentSize, func(start, end time.Time) error {
_, err := sc.load(start, end, sc.location)
- if errors.Is(err, ErrEndOfSegment) {
+ if errors.Is(err, errEndOfSegment) {
return nil
}
return err
@@ -207,7 +207,7 @@ func (sc *segmentController) create(start time.Time) (*segment, error) {
next = s
}
}
- stdEnd := sc.segmentSize.NextTime(start)
+ stdEnd := sc.segmentSize.nextTime(start)
var end time.Time
if next != nil && next.Start.Before(stdEnd) {
end = next.Start
diff --git a/banyand/tsdb/series.go b/banyand/tsdb/series.go
index 7ea6e66..a872c82 100644
--- a/banyand/tsdb/series.go
+++ b/banyand/tsdb/series.go
@@ -33,11 +33,14 @@ import (
)
var (
+ // ErrEmptySeriesSpan hints there is no any data blocks based on the input time range.
ErrEmptySeriesSpan = errors.New("there is no data in such time range")
- ErrItemIDMalformed = errors.New("serialized item id is malformed")
- ErrBlockAbsent = errors.New("block is absent")
+ errItemIDMalformed = errors.New("serialized item id is malformed")
+ errBlockAbsent = errors.New("block is absent")
)
+// GlobalItemID is the top level identity of an item.
+// The item could be retrieved by a GlobalItemID in a tsdb.
type GlobalItemID struct {
ShardID common.ShardID
segID SectionID
@@ -46,7 +49,7 @@ type GlobalItemID struct {
ID common.ItemID
}
-func (i *GlobalItemID) Marshal() []byte {
+func (i *GlobalItemID) marshal() []byte {
return bytes.Join([][]byte{
convert.Uint32ToBytes(uint32(i.ShardID)),
sectionIDToBytes(i.segID),
@@ -56,9 +59,9 @@ func (i *GlobalItemID) Marshal() []byte {
}, nil)
}
-func (i *GlobalItemID) UnMarshal(data []byte) error {
+func (i *GlobalItemID) unMarshal(data []byte) error {
if len(data) != 4+4+4+8+8 {
- return ErrItemIDMalformed
+ return errItemIDMalformed
}
var offset int
i.ShardID = common.ShardID(convert.BytesToUint32(data[offset : offset+4]))
@@ -71,6 +74,8 @@ func (i *GlobalItemID) UnMarshal(data []byte) error {
return nil
}
+// Series denotes a series of data points group by a common.SeriesID
+// common.SeriesID is encoded by a entity defined by Stream or Measure.
type Series interface {
ID() common.SeriesID
Span(ctx context.Context, timeRange timestamp.TimeRange) (SeriesSpan, error)
@@ -79,6 +84,7 @@ type Series interface {
String() string
}
+// SeriesSpan is a span in a time series. It contains data blocks in such time range.
type SeriesSpan interface {
io.Closer
WriterBuilder() WriterBuilder
@@ -101,7 +107,7 @@ func (s *series) Get(ctx context.Context, id GlobalItemID) (Item, io.Closer, err
return nil, nil, err
}
if b == nil {
- return nil, nil, errors.WithMessagef(ErrBlockAbsent, "id: %v", id)
+ return nil, nil, errors.WithMessagef(errBlockAbsent, "id: %v", id)
}
return &item{
data: b.dataReader(),
@@ -191,7 +197,7 @@ type seriesSpan struct {
l *logger.Logger
timeRange timestamp.TimeRange
series string
- blocks []BlockDelegate
+ blocks []blockDelegate
seriesID common.SeriesID
shardID common.ShardID
}
@@ -211,7 +217,7 @@ func (s *seriesSpan) SeekerBuilder() SeekerBuilder {
return newSeekerBuilder(s)
}
-func newSeriesSpan(ctx context.Context, timeRange timestamp.TimeRange, blocks []BlockDelegate, id common.SeriesID, series string, shardID common.ShardID) *seriesSpan {
+func newSeriesSpan(ctx context.Context, timeRange timestamp.TimeRange, blocks []blockDelegate, id common.SeriesID, series string, shardID common.ShardID) *seriesSpan {
s := &seriesSpan{
blocks: blocks,
seriesID: id,
diff --git a/banyand/tsdb/series_seek.go b/banyand/tsdb/series_seek.go
index db66d98..28147f7 100644
--- a/banyand/tsdb/series_seek.go
+++ b/banyand/tsdb/series_seek.go
@@ -34,12 +34,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
+// Iterator allows iterating a series in a time span.
type Iterator interface {
Next() bool
Val() Item
Close() error
}
+// Item allows retrieving raw data from an item.
type Item interface {
Family(family []byte) ([]byte, error)
PrintContext(l *logger.Logger, family []byte, n int)
@@ -49,6 +51,7 @@ type Item interface {
Time() uint64
}
+// SeekerBuilder a helper to build a Seeker.
type SeekerBuilder interface {
Filter(predicator index.Filter) SeekerBuilder
OrderByIndex(indexRule *databasev1.IndexRule, order modelv1.Sort) SeekerBuilder
@@ -56,6 +59,7 @@ type SeekerBuilder interface {
Build() (Seeker, error)
}
+// Seeker allows searching data in a Database.
type Seeker interface {
Seek() ([]Iterator, error)
}
diff --git a/banyand/tsdb/series_seek_filter.go b/banyand/tsdb/series_seek_filter.go
index 7e8cd4a..64d7ed1 100644
--- a/banyand/tsdb/series_seek_filter.go
+++ b/banyand/tsdb/series_seek_filter.go
@@ -24,14 +24,14 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index"
)
-var ErrUnsupportedIndexRule = errors.New("the index rule is not supported")
+var errUnsupportedIndexRule = errors.New("the index rule is not supported")
func (s *seekerBuilder) Filter(predicator index.Filter) SeekerBuilder {
s.predicator = predicator
return s
}
-func (s *seekerBuilder) buildIndexFilter(block BlockDelegate) (filterFn, error) {
+func (s *seekerBuilder) buildIndexFilter(block blockDelegate) (filterFn, error) {
if s.predicator == nil {
return nil, nil
}
@@ -42,7 +42,7 @@ func (s *seekerBuilder) buildIndexFilter(block BlockDelegate) (filterFn, error)
case databasev1.IndexRule_TYPE_TREE:
return block.lsmIndexReader(), nil
default:
- return nil, ErrUnsupportedIndexRule
+ return nil, errUnsupportedIndexRule
}
}, s.seriesSpan.seriesID)
if err != nil {
diff --git a/banyand/tsdb/series_seek_sort.go b/banyand/tsdb/series_seek_sort.go
index d56cacf..40d05b2 100644
--- a/banyand/tsdb/series_seek_sort.go
+++ b/banyand/tsdb/series_seek_sort.go
@@ -36,7 +36,7 @@ import (
)
var (
- ErrUnspecifiedIndexType = errors.New("Unspecified index type")
+ errUnspecifiedIndexType = errors.New("Unspecified index type")
emptyFilters = make([]filterFn, 0)
)
@@ -89,7 +89,7 @@ func (s *seekerBuilder) buildSeriesByIndex() (series []Iterator, err error) {
case databasev1.IndexRule_TYPE_INVERTED:
inner, err = b.invertedIndexReader().Iterator(fieldKey, s.rangeOptsForSorting, s.order)
case databasev1.IndexRule_TYPE_UNSPECIFIED:
- return nil, errors.WithMessagef(ErrUnspecifiedIndexType, "index rule:%v", s.indexRuleForSorting)
+ return nil, errors.WithMessagef(errUnspecifiedIndexType, "index rule:%v", s.indexRuleForSorting)
}
if err != nil {
return nil, err
diff --git a/banyand/tsdb/series_write.go b/banyand/tsdb/series_write.go
index b6da92c..f903566 100644
--- a/banyand/tsdb/series_write.go
+++ b/banyand/tsdb/series_write.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/index"
)
+// WriterBuilder is a helper to build a Writer.
type WriterBuilder interface {
Family(name []byte, val []byte) WriterBuilder
Time(ts time.Time) WriterBuilder
@@ -36,6 +37,7 @@ type WriterBuilder interface {
Build() (Writer, error)
}
+// Writer allow ingesting data into a tsdb.
type Writer interface {
IndexWriter
Write() (GlobalItemID, error)
@@ -47,7 +49,7 @@ var _ WriterBuilder = (*writerBuilder)(nil)
type writerBuilder struct {
series *seriesSpan
- block BlockDelegate
+ block blockDelegate
values []struct {
family []byte
val []byte
@@ -84,26 +86,25 @@ func (w *writerBuilder) Val(val []byte) WriterBuilder {
}
var (
- ErrNoTime = errors.New("no time specified")
- ErrNoVal = errors.New("no value specified")
+ errNoTime = errors.New("no time specified")
+ errNoVal = errors.New("no value specified")
+ errDuplicatedFamily = errors.New("duplicated family")
)
-var ErrDuplicatedFamily = errors.New("duplicated family")
-
func (w *writerBuilder) Build() (Writer, error) {
if w.block == nil {
- return nil, errors.WithMessagef(ErrNoTime, "ts:%v", w.ts)
+ return nil, errors.WithMessagef(errNoTime, "ts:%v", w.ts)
}
if len(w.values) < 1 {
- return nil, errors.WithStack(ErrNoVal)
+ return nil, errors.WithStack(errNoVal)
}
for i, value := range w.values {
for j := i + 1; j < len(w.values); j++ {
if value.family == nil && w.values[j].family == nil {
- return nil, errors.Wrap(ErrDuplicatedFamily, "default family")
+ return nil, errors.Wrap(errDuplicatedFamily, "default family")
}
if bytes.Equal(value.family, w.values[j].family) {
- return nil, errors.Wrapf(ErrDuplicatedFamily, "family:%s", value.family)
+ return nil, errors.Wrapf(errDuplicatedFamily, "family:%s", value.family)
}
}
}
@@ -133,7 +134,7 @@ var _ Writer = (*writer)(nil)
type writer struct {
ts time.Time
- block BlockDelegate
+ block blockDelegate
itemID *GlobalItemID
columns []struct {
family []byte
diff --git a/banyand/tsdb/seriesdb.go b/banyand/tsdb/seriesdb.go
index aac8e55..88acc8d 100644
--- a/banyand/tsdb/seriesdb.go
+++ b/banyand/tsdb/seriesdb.go
@@ -49,12 +49,17 @@ var (
zeroIntBytes = convert.Uint64ToBytes(0)
)
+// AnyEntry is the `*` for a regular expression. It could match "any" Entry in an Entity.
var AnyEntry = Entry(nil)
+// Entry is an element in an Entity.
type Entry []byte
+// Entity denotes an identity of a Series.
+// It defined by Stream or Measure schema.
type Entity []Entry
+// Marshal encodes an Entity to bytes.
func (e Entity) Marshal() []byte {
data := make([][]byte, len(e))
for i, entry := range e {
@@ -63,18 +68,21 @@ func (e Entity) Marshal() []byte {
return bytes.Join(data, nil)
}
+// Prepend inserts an Entry before the first Entry as the prefix.
func (e Entity) Prepend(entry Entry) Entity {
d := e
d = append(Entity{entry}, d...)
return d
}
+// Copy an Entity deeply.
func (e Entity) Copy() Entity {
a := make(Entity, len(e))
copy(a, e)
return a
}
+// NewEntity return an Entity with an fixed length.
func NewEntity(length int) Entity {
e := make(Entity, length)
for i := 0; i < length; i++ {
@@ -83,18 +91,23 @@ func NewEntity(length int) Entity {
return e
}
+// EntityValue represents the value of a tag which is a part of an entity.
type EntityValue *modelv1.TagValue
+// EntityValueToEntry transforms EntityValue to Entry.
func EntityValueToEntry(ev EntityValue) (Entry, error) {
return pbv1.MarshalTagValue(ev)
}
+// EntityValues is the encoded Entity.
type EntityValues []EntityValue
+// Prepend inserts an EntityValue before the first EntityValue as the prefix.
func (evs EntityValues) Prepend(scope EntityValue) EntityValues {
return append(EntityValues{scope}, evs...)
}
+// Encode EntityValues to tag values.
func (evs EntityValues) Encode() (result []*modelv1.TagValue) {
for _, v := range evs {
result = append(result, v)
@@ -102,6 +115,7 @@ func (evs EntityValues) Encode() (result []*modelv1.TagValue) {
return
}
+// ToEntity transforms EntityValues to Entity.
func (evs EntityValues) ToEntity() (result Entity, err error) {
for _, v := range evs {
entry, errMarshal := EntityValueToEntry(v)
@@ -113,6 +127,7 @@ func (evs EntityValues) ToEntity() (result Entity, err error) {
return
}
+// String outputs the string represent of an EntityValue.
func (evs EntityValues) String() string {
var strBuilder strings.Builder
vv := evs.Encode()
@@ -125,6 +140,7 @@ func (evs EntityValues) String() string {
return strBuilder.String()
}
+// DecodeEntityValues decodes tag values to EntityValues.
func DecodeEntityValues(tvv []*modelv1.TagValue) (result EntityValues) {
for _, tv := range tvv {
result = append(result, tv)
@@ -132,14 +148,17 @@ func DecodeEntityValues(tvv []*modelv1.TagValue) (result EntityValues) {
return
}
+// StrValue returns an EntityValue which wraps a string value.
func StrValue(v string) EntityValue {
return &modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value: v}}}
}
+// Int64Value returns an EntityValue which wraps a int64 value.
func Int64Value(v int64) EntityValue {
return &modelv1.TagValue{Value: &modelv1.TagValue_Int{Int: &modelv1.Int{Value: v}}}
}
+// MarshalEntityValues encodes EntityValues to bytes.
func MarshalEntityValues(evs EntityValues) ([]byte, error) {
data := &modelv1.TagFamilyForWrite{}
for _, v := range evs {
@@ -148,6 +167,7 @@ func MarshalEntityValues(evs EntityValues) ([]byte, error) {
return proto.Marshal(data)
}
+// UnmarshalEntityValues decodes EntityValues from bytes.
func UnmarshalEntityValues(evs []byte) (result EntityValues, err error) {
data := &modelv1.TagFamilyForWrite{}
result = make(EntityValues, len(data.Tags))
@@ -160,6 +180,8 @@ func UnmarshalEntityValues(evs []byte) (result EntityValues, err error) {
return
}
+// Path denotes a expression to match a Series.
+// It supports the fuzzy matching more than EQ by setting an entry to AnyEntry.
type Path struct {
prefix []byte
seekKey []byte
@@ -169,7 +191,8 @@ type Path struct {
offset int
}
-func NewPath(entries []Entry) Path {
+// NewPath return a Path with a matching expression.
+func NewPath(matchingExpression []Entry) Path {
p := Path{
seekKey: make([]byte, 0),
mask: make([]byte, 0),
@@ -177,7 +200,7 @@ func NewPath(entries []Entry) Path {
}
var encounterAny bool
- for _, e := range entries {
+ for _, e := range matchingExpression {
if e == nil {
encounterAny = true
p.mask = append(p.mask, zeroIntBytes...)
@@ -207,7 +230,7 @@ func (p *Path) extractPrefix() {
}
}
-func (p Path) Prepend(entry Entry) Path {
+func (p Path) prepend(entry Entry) Path {
e := Hash(entry)
p.template = prepend(p.template, e)
p.offset += len(e)
@@ -223,6 +246,7 @@ func prepend(src []byte, entry []byte) []byte {
return dst
}
+// SeriesDatabase allows retrieving series.
type SeriesDatabase interface {
observability.Observable
io.Closer
@@ -233,9 +257,9 @@ type SeriesDatabase interface {
type blockDatabase interface {
shardID() common.ShardID
- span(ctx context.Context, timeRange timestamp.TimeRange) ([]BlockDelegate, error)
- create(ctx context.Context, ts time.Time) (BlockDelegate, error)
- block(ctx context.Context, id GlobalItemID) (BlockDelegate, error)
+ span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error)
+ create(ctx context.Context, ts time.Time) (blockDelegate, error)
+ block(ctx context.Context, id GlobalItemID) (blockDelegate, error)
}
var (
@@ -272,7 +296,7 @@ func (s *seriesDB) GetByID(id common.SeriesID) (Series, error) {
return newSeries(s.context(), id, series, s), nil
}
-func (s *seriesDB) block(ctx context.Context, id GlobalItemID) (BlockDelegate, error) {
+func (s *seriesDB) block(ctx context.Context, id GlobalItemID) (blockDelegate, error) {
seg := s.segCtrl.get(id.segID)
if seg == nil {
return nil, nil
@@ -410,8 +434,8 @@ func (s *seriesDB) List(ctx context.Context, path Path) (SeriesList, error) {
return result, err
}
-func (s *seriesDB) span(ctx context.Context, timeRange timestamp.TimeRange) ([]BlockDelegate, error) {
- result := make([]BlockDelegate, 0)
+func (s *seriesDB) span(ctx context.Context, timeRange timestamp.TimeRange) ([]blockDelegate, error) {
+ result := make([]blockDelegate, 0)
for _, s := range s.segCtrl.span(timeRange) {
dd, err := s.blockController.span(ctx, timeRange)
if err != nil {
@@ -425,7 +449,7 @@ func (s *seriesDB) span(ctx context.Context, timeRange timestamp.TimeRange) ([]B
return result, nil
}
-func (s *seriesDB) create(ctx context.Context, ts time.Time) (BlockDelegate, error) {
+func (s *seriesDB) create(ctx context.Context, ts time.Time) (blockDelegate, error) {
s.Lock()
defer s.Unlock()
timeRange := timestamp.NewInclusiveTimeRange(ts, ts)
@@ -508,10 +532,12 @@ func HashEntity(entity Entity) []byte {
return result
}
+// SeriesID transforms Entity to common.SeriesID.
func SeriesID(entity Entity) common.SeriesID {
return common.SeriesID(convert.Hash(HashEntity(entity)))
}
+// Hash encode Entry to 8 bytes.
func Hash(entry []byte) []byte {
return convert.Uint64ToBytes(convert.Hash(entry))
}
@@ -520,6 +546,7 @@ func bytesToSeriesID(data []byte) common.SeriesID {
return common.SeriesID(convert.BytesToUint64(data))
}
+// SeriesList is a collection of Series.
type SeriesList []Series
func (a SeriesList) Len() int {
@@ -534,6 +561,7 @@ func (a SeriesList) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
+// Merge other SeriesList with this one to create a new SeriesList.
func (a SeriesList) Merge(other SeriesList) SeriesList {
if len(other) == 0 {
return a
diff --git a/banyand/tsdb/seriesdb_test.go b/banyand/tsdb/seriesdb_test.go
index 4d232f5..129f320 100644
--- a/banyand/tsdb/seriesdb_test.go
+++ b/banyand/tsdb/seriesdb_test.go
@@ -342,7 +342,7 @@ func TestNewPath(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
got := NewPath(tt.entity)
if tt.scope != nil {
- got = got.Prepend(tt.scope)
+ got = got.prepend(tt.scope)
}
tester.Equal(tt.want, got)
})
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index f89b2d8..07b89a7 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -54,6 +54,7 @@ type shard struct {
id common.ShardID
}
+// OpenShard returns an existed Shard or create a new one if not existed.
func OpenShard(ctx context.Context, id common.ShardID,
root string, segmentSize, blockSize, ttl IntervalRule, openedBlockSize, maxOpenedBlockSize int,
) (Shard, error) {
@@ -178,8 +179,10 @@ func (s *shard) Close() (err error) {
return err
}
+// IntervalUnit denotes the unit of a time point.
type IntervalUnit int
+// Available IntervalUnits. HOUR and DAY are adequate for the APM scenario.
const (
HOUR IntervalUnit = iota
DAY
@@ -195,12 +198,13 @@ func (iu IntervalUnit) String() string {
panic("invalid interval unit")
}
+// IntervalRule defines a length of two points in time.
type IntervalRule struct {
Unit IntervalUnit
Num int
}
-func (ir IntervalRule) NextTime(current time.Time) time.Time {
+func (ir IntervalRule) nextTime(current time.Time) time.Time {
switch ir.Unit {
case HOUR:
return current.Add(time.Hour * time.Duration(ir.Num))
@@ -210,17 +214,7 @@ func (ir IntervalRule) NextTime(current time.Time) time.Time {
panic("invalid interval unit")
}
-func (ir IntervalRule) PreviousTime(current time.Time) time.Time {
- switch ir.Unit {
- case HOUR:
- return current.Add(-time.Hour * time.Duration(ir.Num))
- case DAY:
- return current.AddDate(0, 0, -ir.Num)
- }
- panic("invalid interval unit")
-}
-
-func (ir IntervalRule) EstimatedDuration() time.Duration {
+func (ir IntervalRule) estimatedDuration() time.Duration {
switch ir.Unit {
case HOUR:
return time.Hour * time.Duration(ir.Num)
@@ -236,7 +230,7 @@ type parser interface {
func loadSections(root string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
var startTimeLst []time.Time
- if err := WalkDir(
+ if err := walkDir(
root,
segPathPrefix,
func(suffix string) error {
@@ -255,7 +249,7 @@ func loadSections(root string, parser parser, intervalRule IntervalRule, loadFn
if i < len(startTimeLst)-1 {
end = startTimeLst[i+1]
} else {
- end = intervalRule.NextTime(start)
+ end = intervalRule.nextTime(start)
}
if err := loadFn(start, end); err != nil {
return err
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index 70546db..8f6ad3a 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -15,6 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+// Package tsdb implements a time-series-based storage engine.
+// It provides:
+// - Partition data based on a time axis.
+// - Sharding data based on a series id which represents a unique entity of stream/measure
+// - Retrieving data based on index.Filter.
+// - Cleaning expired data, or the data retention.
package tsdb
import (
@@ -55,23 +61,27 @@ const (
)
var (
- ErrInvalidShardID = errors.New("invalid shard id")
- ErrOpenDatabase = errors.New("fails to open the database")
+ errInvalidShardID = errors.New("invalid shard id")
+ errOpenDatabase = errors.New("fails to open the database")
optionsKey = contextOptionsKey{}
)
type contextOptionsKey struct{}
+// Supplier allows getting a tsdb's runtime.
type Supplier interface {
SupplyTSDB() Database
}
+
+// Database allows listing and getting shard details.
type Database interface {
io.Closer
Shards() []Shard
Shard(id common.ShardID) (Shard, error)
}
+// Shard allows accessing data of tsdb.
type Shard interface {
io.Closer
ID() common.ShardID
@@ -84,6 +94,7 @@ type Shard interface {
var _ Database = (*database)(nil)
+// DatabaseOpts wraps options to create a tsdb.
type DatabaseOpts struct {
EncodingMethod EncodingMethod
Location string
@@ -97,14 +108,18 @@ type DatabaseOpts struct {
EnableGlobalIndex bool
}
+// EncodingMethod wraps encoder/decoder pools to flush/compact data on disk.
type EncodingMethod struct {
EncoderPool encoding.SeriesEncoderPool
DecoderPool encoding.SeriesDecoderPool
}
type (
+ // SectionID is the kind of a block/segment.
SectionID uint32
- BlockID struct {
+
+ // BlockID is the identity of a block in a shard.
+ BlockID struct {
SegID SectionID
BlockID SectionID
}
@@ -114,6 +129,7 @@ func (b BlockID) String() string {
return fmt.Sprintf("BlockID-%d-%d", parseSuffix(b.SegID), parseSuffix(b.BlockID))
}
+// GenerateInternalID returns a identity of a section(segment or block) based on IntervalRule.
func GenerateInternalID(unit IntervalUnit, suffix int) SectionID {
return SectionID(unit)<<31 | ((SectionID(suffix) << 1) >> 1)
}
@@ -131,11 +147,14 @@ func readSectionID(data []byte, offset int) (SectionID, int) {
return SectionID(convert.BytesToUint32(data[offset:end])), end
}
+// BlockState is a sample of a block's runtime state.
type BlockState struct {
TimeRange timestamp.TimeRange
ID BlockID
Closed bool
}
+
+// ShardState is a sample of a shard's runtime state.
type ShardState struct {
Blocks []BlockState
OpenBlocks []BlockID
@@ -159,7 +178,7 @@ func (d *database) Shards() []Shard {
func (d *database) Shard(id common.ShardID) (Shard, error) {
if uint(id) >= uint(len(d.sLst)) {
- return nil, ErrInvalidShardID
+ return nil, errInvalidShardID
}
return d.sLst[id], nil
}
@@ -175,24 +194,26 @@ func (d *database) Close() error {
return err
}
+// OpenDatabase returns a new tsdb runtime. This constructor will create a new database if it's absent,
+// or load an existing one.
func OpenDatabase(ctx context.Context, opts DatabaseOpts) (Database, error) {
if opts.EncodingMethod.EncoderPool == nil || opts.EncodingMethod.DecoderPool == nil {
- return nil, errors.Wrap(ErrOpenDatabase, "encoding method is absent")
+ return nil, errors.Wrap(errOpenDatabase, "encoding method is absent")
}
if _, err := mkdir(opts.Location); err != nil {
return nil, err
}
if opts.SegmentInterval.Num == 0 {
- return nil, errors.Wrap(ErrOpenDatabase, "segment interval is absent")
+ return nil, errors.Wrap(errOpenDatabase, "segment interval is absent")
}
if opts.BlockInterval.Num == 0 {
- return nil, errors.Wrap(ErrOpenDatabase, "block interval is absent")
+ return nil, errors.Wrap(errOpenDatabase, "block interval is absent")
}
- if opts.BlockInterval.EstimatedDuration() > opts.SegmentInterval.EstimatedDuration() {
- return nil, errors.Wrapf(ErrOpenDatabase, "the block size is bigger than the segment size")
+ if opts.BlockInterval.estimatedDuration() > opts.SegmentInterval.estimatedDuration() {
+ return nil, errors.Wrapf(errOpenDatabase, "the block size is bigger than the segment size")
}
if opts.TTL.Num == 0 {
- return nil, errors.Wrap(ErrOpenDatabase, "ttl is absent")
+ return nil, errors.Wrap(errOpenDatabase, "ttl is absent")
}
db := &database{
location: opts.Location,
@@ -242,7 +263,7 @@ func loadDatabase(ctx context.Context, db *database) (Database, error) {
// TODO: open the manifest file
db.Lock()
defer db.Unlock()
- err := WalkDir(db.location, shardPathPrefix, func(suffix string) error {
+ err := walkDir(db.location, shardPathPrefix, func(suffix string) error {
shardID, err := strconv.Atoi(suffix)
if err != nil {
return err
@@ -281,9 +302,9 @@ func loadDatabase(ctx context.Context, db *database) (Database, error) {
return db, nil
}
-type WalkFn func(suffix string) error
+type walkFn func(suffix string) error
-func WalkDir(root, prefix string, walkFn WalkFn) error {
+func walkDir(root, prefix string, wf walkFn) error {
files, err := os.ReadDir(root)
if err != nil {
return errors.Wrapf(err, "failed to walk the database path: %s", root)
@@ -293,7 +314,7 @@ func WalkDir(root, prefix string, walkFn WalkFn) error {
continue
}
segs := strings.Split(f.Name(), "-")
- errWalk := walkFn(segs[len(segs)-1])
+ errWalk := wf(segs[len(segs)-1])
if errWalk != nil {
return errors.WithMessagef(errWalk, "failed to load: %s", f.Name())
}
diff --git a/pkg/schema/metadata.go b/pkg/schema/metadata.go
index 1b2f979..daa2859 100644
--- a/pkg/schema/metadata.go
+++ b/pkg/schema/metadata.go
@@ -88,7 +88,7 @@ type Resource interface {
type ResourceSupplier interface {
OpenResource(shardNum uint32, db tsdb.Supplier, spec ResourceSpec) (Resource, error)
- ResourceSchema(repo metadata.Repo, metdata *commonv1.Metadata) (ResourceSchema, error)
+ ResourceSchema(metdata *commonv1.Metadata) (ResourceSchema, error)
OpenDB(groupSchema *commonv1.Group) (tsdb.Database, error)
}
@@ -287,7 +287,7 @@ func (sr *schemaRepo) storeResource(metadata *commonv1.Metadata) (Resource, erro
return nil, errors.WithMessagef(err, "create unknown group:%s", metadata.Group)
}
}
- stm, err := sr.resourceSupplier.ResourceSchema(sr.metadata, metadata)
+ stm, err := sr.resourceSupplier.ResourceSchema(metadata)
if err != nil {
return nil, errors.WithMessage(err, "fails to get the resource")
}
diff --git a/pkg/test/setup/setup.go b/pkg/test/setup/setup.go
index 4235604..e8b3063 100644
--- a/pkg/test/setup/setup.go
+++ b/pkg/test/setup/setup.go
@@ -82,7 +82,7 @@ func modules(flags []string) func() {
measureSvc, err := measure.NewService(context.TODO(), metaSvc, repo, pipeline)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
// Init `Query` module
- q, err := query.NewExecutor(context.TODO(), streamSvc, measureSvc, metaSvc, repo, pipeline)
+ q, err := query.NewService(context.TODO(), streamSvc, measureSvc, metaSvc, repo, pipeline)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
tcp := grpc.NewServer(context.TODO(), pipeline, repo, metaSvc)
httpServer := http.NewService()