You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by GitBox <gi...@apache.org> on 2021/08/03 14:43:54 UTC
[GitHub] [skywalking-banyandb] hanahmily opened a new pull request #26: Introduce index module with memtable only
hanahmily opened a new pull request #26:
URL: https://github.com/apache/skywalking-banyandb/pull/26
In this PR, I introduce the index module, which only contains the memtable. Other components, for instance, on-disk table and kv storage, will come in follow-up PRs.
@lujiajing1126 The query module can integrate with the index module once this gets merged. The incomplete index module can provide full features to other modules, but don't write a massive volume data into the index module because all index data is laid in the main memory.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@skywalking.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [skywalking-banyandb] hanahmily commented on a change in pull request #26: Introduce index module with memtable only
Posted by GitBox <gi...@apache.org>.
hanahmily commented on a change in pull request #26:
URL: https://github.com/apache/skywalking-banyandb/pull/26#discussion_r682251799
##########
File path: banyand/index/search.go
##########
@@ -0,0 +1,359 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package index
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "strings"
+
+ "github.com/pkg/errors"
+ "github.com/rs/zerolog"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ apiv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/v1"
+ "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/posting"
+ "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
+)
+
+var ErrNotRangeOperation = errors.New("this is not an range operation")
+
+type executable interface {
+ execute() (posting.List, error)
+}
+
+type searchTree interface {
+ executable
+}
+
+func (s *service) Search(series common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error) {
+ sd, err := s.getShard(series, shardID)
+ if err != nil {
+ return nil, err
+ }
+ store := sd.store
+ searcher, hasData := store.Window(startTime, endTime)
+ if !hasData {
+ return roaring.EmptyPostingList, nil
+ }
+ tree, errBuild := buildSearchTree(searcher, indexObjectName, conditions)
+ if errBuild != nil {
+ return nil, err
+ }
+ if s.log.Should(zerolog.DebugLevel) {
Review comment:
will remove it. I have found `Event.Interface` to print `tree` instead of marshaling explicitly, so we don't need the `should` method to reduce the overhead of JSON marshaling.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@skywalking.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [skywalking-banyandb] hanahmily merged pull request #26: Introduce index module with memtable only
Posted by GitBox <gi...@apache.org>.
hanahmily merged pull request #26:
URL: https://github.com/apache/skywalking-banyandb/pull/26
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@skywalking.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [skywalking-banyandb] lujiajing1126 commented on a change in pull request #26: Introduce index module with memtable only
Posted by GitBox <gi...@apache.org>.
lujiajing1126 commented on a change in pull request #26:
URL: https://github.com/apache/skywalking-banyandb/pull/26#discussion_r682217365
##########
File path: banyand/index/index.go
##########
@@ -19,36 +19,230 @@ package index
import (
"context"
+ "sync"
+
+ "github.com/pkg/errors"
+ "go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/api/event"
apiv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/v1"
"github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/posting"
"github.com/apache/skywalking-banyandb/pkg/run"
)
+var (
+ ErrShardNotFound = errors.New("series doesn't exist")
+ ErrTraceSeriesNotFound = errors.New("trace series not found")
+ ErrUnknownField = errors.New("the field is unknown")
+)
+
type Condition struct {
Key string
Values [][]byte
Op apiv1.PairQuery_BinaryOp
}
-//go:generate mockgen -destination=./index_mock.go -package=index . Repo
+type Field struct {
+ ChunkID common.ChunkID
+ Name string
+ Value []byte
+}
+
+//go:generate mockgen -destination=./index_mock.go -package=index . Service
type Repo interface {
Search(seriesMeta common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error)
+ Insert(seriesMeta common.Metadata, shardID uint, fields *Field) error
}
type Builder interface {
- run.Config
run.PreRunner
+ run.Service
}
type Service interface {
Repo
Builder
}
-func NewService(ctx context.Context, repo discovery.ServiceRepo, pipeline queue.Queue) (Service, error) {
- return nil, nil
+type series struct {
+ repo map[uint]*shard
+}
+
+type shard struct {
+ meta map[string][]*apiv1.IndexObject
+ store tsdb.GlobalStore
+}
+
+type service struct {
+ meta *indexMeta
+ log *logger.Logger
+ repo discovery.ServiceRepo
+ stopCh chan struct{}
+ indexRuleListener *indexRuleListener
+}
+
+func NewService(_ context.Context, repo discovery.ServiceRepo) (Service, error) {
+ svc := &service{
+ repo: repo,
+ indexRuleListener: &indexRuleListener{},
+ }
+ svc.meta = &indexMeta{
+ meta: make(map[string]*series),
+ }
+ svc.indexRuleListener.indexMeta = svc.meta
+ svc.indexRuleListener.closeFunc = func() {
+ svc.stopCh <- struct{}{}
+ }
+ return svc, nil
+}
+
+func (s *service) Insert(series common.Metadata, shardID uint, field *Field) error {
+ sd, err := s.getShard(series, shardID)
+ if err != nil {
+ return err
+ }
+ objects, ok := sd.meta[field.Name]
+ if !ok {
+ return ErrUnknownField
+ }
+ for _, object := range objects {
+ err = multierr.Append(err, sd.store.Insert(&tsdb.Field{
+ Name: []byte(compositeFieldID(object.GetName(), field.Name)),
+ Value: field.Value,
+ }, field.ChunkID))
+ }
+ return err
+}
+
+func (s *service) getShard(series common.Metadata, shardID uint) (*shard, error) {
+ id := compositeSeriesID(series.Spec)
+ ss, ok := s.meta.meta[id]
Review comment:
use `s.get(series *apiv1.Metadata)` in order to make use of read lock?
##########
File path: banyand/index/search.go
##########
@@ -0,0 +1,359 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package index
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "strings"
+
+ "github.com/pkg/errors"
+ "github.com/rs/zerolog"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ apiv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/v1"
+ "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
+ "github.com/apache/skywalking-banyandb/pkg/bytes"
+ "github.com/apache/skywalking-banyandb/pkg/posting"
+ "github.com/apache/skywalking-banyandb/pkg/posting/roaring"
+)
+
+var ErrNotRangeOperation = errors.New("this is not an range operation")
+
+type executable interface {
+ execute() (posting.List, error)
+}
+
+type searchTree interface {
+ executable
+}
+
+func (s *service) Search(series common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error) {
+ sd, err := s.getShard(series, shardID)
+ if err != nil {
+ return nil, err
+ }
+ store := sd.store
+ searcher, hasData := store.Window(startTime, endTime)
+ if !hasData {
+ return roaring.EmptyPostingList, nil
+ }
+ tree, errBuild := buildSearchTree(searcher, indexObjectName, conditions)
+ if errBuild != nil {
+ return nil, err
+ }
+ if s.log.Should(zerolog.DebugLevel) {
Review comment:
why we need to check level here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@skywalking.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [skywalking-banyandb] hanahmily commented on a change in pull request #26: Introduce index module with memtable only
Posted by GitBox <gi...@apache.org>.
hanahmily commented on a change in pull request #26:
URL: https://github.com/apache/skywalking-banyandb/pull/26#discussion_r682252772
##########
File path: banyand/index/index.go
##########
@@ -19,36 +19,230 @@ package index
import (
"context"
+ "sync"
+
+ "github.com/pkg/errors"
+ "go.uber.org/multierr"
"github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/api/event"
apiv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/v1"
"github.com/apache/skywalking-banyandb/banyand/discovery"
- "github.com/apache/skywalking-banyandb/banyand/queue"
+ "github.com/apache/skywalking-banyandb/banyand/index/tsdb"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/posting"
"github.com/apache/skywalking-banyandb/pkg/run"
)
+var (
+ ErrShardNotFound = errors.New("series doesn't exist")
+ ErrTraceSeriesNotFound = errors.New("trace series not found")
+ ErrUnknownField = errors.New("the field is unknown")
+)
+
type Condition struct {
Key string
Values [][]byte
Op apiv1.PairQuery_BinaryOp
}
-//go:generate mockgen -destination=./index_mock.go -package=index . Repo
+type Field struct {
+ ChunkID common.ChunkID
+ Name string
+ Value []byte
+}
+
+//go:generate mockgen -destination=./index_mock.go -package=index . Service
type Repo interface {
Search(seriesMeta common.Metadata, shardID uint, startTime, endTime uint64, indexObjectName string, conditions []Condition) (posting.List, error)
+ Insert(seriesMeta common.Metadata, shardID uint, fields *Field) error
}
type Builder interface {
- run.Config
run.PreRunner
+ run.Service
}
type Service interface {
Repo
Builder
}
-func NewService(ctx context.Context, repo discovery.ServiceRepo, pipeline queue.Queue) (Service, error) {
- return nil, nil
+type series struct {
+ repo map[uint]*shard
+}
+
+type shard struct {
+ meta map[string][]*apiv1.IndexObject
+ store tsdb.GlobalStore
+}
+
+type service struct {
+ meta *indexMeta
+ log *logger.Logger
+ repo discovery.ServiceRepo
+ stopCh chan struct{}
+ indexRuleListener *indexRuleListener
+}
+
+func NewService(_ context.Context, repo discovery.ServiceRepo) (Service, error) {
+ svc := &service{
+ repo: repo,
+ indexRuleListener: &indexRuleListener{},
+ }
+ svc.meta = &indexMeta{
+ meta: make(map[string]*series),
+ }
+ svc.indexRuleListener.indexMeta = svc.meta
+ svc.indexRuleListener.closeFunc = func() {
+ svc.stopCh <- struct{}{}
+ }
+ return svc, nil
+}
+
+func (s *service) Insert(series common.Metadata, shardID uint, field *Field) error {
+ sd, err := s.getShard(series, shardID)
+ if err != nil {
+ return err
+ }
+ objects, ok := sd.meta[field.Name]
+ if !ok {
+ return ErrUnknownField
+ }
+ for _, object := range objects {
+ err = multierr.Append(err, sd.store.Insert(&tsdb.Field{
+ Name: []byte(compositeFieldID(object.GetName(), field.Name)),
+ Value: field.Value,
+ }, field.ChunkID))
+ }
+ return err
+}
+
+func (s *service) getShard(series common.Metadata, shardID uint) (*shard, error) {
+ id := compositeSeriesID(series.Spec)
+ ss, ok := s.meta.meta[id]
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@skywalking.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org