You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/01/04 06:25:50 UTC
[skywalking-banyandb] 01/01: Correct the failure of opening existing blocks
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch kv
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit a2ac6f00334027ea2a2a84713f579bbf358f951a
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Jan 4 06:25:07 2023 +0000
Correct the failure of opening existing blocks
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
api/proto/banyandb/measure/v1/query.proto | 2 +-
api/proto/banyandb/model/v1/query.proto | 2 +-
banyand/tsdb/block_ctrl.go | 4 +-
banyand/tsdb/segment_ctrl.go | 4 +-
banyand/tsdb/shard.go | 4 +-
banyand/tsdb/tsdb.go | 12 ++
pkg/query/logical/index_filter.go | 79 ++++++++++--
.../entity.yaml => input/all_only_fields.yaml} | 17 +--
test/cases/measure/data/input/entity.yaml | 2 +-
.../data/input/{entity.yaml => entity_in.yaml} | 6 +-
test/cases/measure/data/want/all_only_fields.yaml | 138 +++++++++++++++++++++
test/cases/measure/data/want/entity.yaml | 4 -
.../data/want/{entity.yaml => entity_in.yaml} | 34 +++--
test/cases/measure/measure.go | 4 +-
14 files changed, 259 insertions(+), 53 deletions(-)
diff --git a/api/proto/banyandb/measure/v1/query.proto b/api/proto/banyandb/measure/v1/query.proto
index 53df387..8350ba8 100644
--- a/api/proto/banyandb/measure/v1/query.proto
+++ b/api/proto/banyandb/measure/v1/query.proto
@@ -57,7 +57,7 @@ message QueryRequest {
// tag_families are indexed.
model.v1.Criteria criteria = 4;
// tag_projection can be used to select tags of the data points in the response
- model.v1.TagProjection tag_projection = 5 [(validate.rules).message.required = true];
+ model.v1.TagProjection tag_projection = 5;
message FieldProjection {
repeated string names = 1;
}
diff --git a/api/proto/banyandb/model/v1/query.proto b/api/proto/banyandb/model/v1/query.proto
index cb4a7a1..2fd325b 100644
--- a/api/proto/banyandb/model/v1/query.proto
+++ b/api/proto/banyandb/model/v1/query.proto
@@ -110,7 +110,7 @@ message TagProjection {
string name = 1;
repeated string tags = 2;
}
- repeated TagFamily tag_families = 1 [(validate.rules).repeated.min_items = 1];
+ repeated TagFamily tag_families = 1;
}
// TimeRange is a range query for uint64,
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
index b1071d3..8d251c4 100644
--- a/banyand/tsdb/block_ctrl.go
+++ b/banyand/tsdb/block_ctrl.go
@@ -226,7 +226,7 @@ func (bc *blockController) closeBlock(ctx context.Context, blockID SectionID) er
func (bc *blockController) open() error {
bc.Lock()
defer bc.Unlock()
- return loadSections(bc.location, bc, bc.blockSize, func(start, end time.Time) error {
+ return loadSections(bc.location, blockPathPrefix, bc, bc.blockSize, func(start, end time.Time) error {
_, err := bc.load(start, end, bc.location)
return err
})
@@ -261,7 +261,7 @@ func (bc *blockController) create(start time.Time) (*block, error) {
if end.After(bc.segTimeRange.End) {
end = bc.segTimeRange.End
}
- _, err := mkdir(blockTemplate, bc.location, bc.Format(start))
+ _, err := mkdirIfNotExist(blockTemplate, bc.location, bc.Format(start))
if err != nil {
return nil, err
}
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
index 98653a2..487295f 100644
--- a/banyand/tsdb/segment_ctrl.go
+++ b/banyand/tsdb/segment_ctrl.go
@@ -185,7 +185,7 @@ func (sc *segmentController) Parse(value string) (time.Time, error) {
func (sc *segmentController) open() error {
sc.Lock()
defer sc.Unlock()
- return loadSections(sc.location, sc, sc.segmentSize, func(start, end time.Time) error {
+ return loadSections(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
_, err := sc.load(start, end, sc.location)
if errors.Is(err, errEndOfSegment) {
return nil
@@ -214,7 +214,7 @@ func (sc *segmentController) create(start time.Time) (*segment, error) {
} else {
end = stdEnd
}
- _, err := mkdir(segTemplate, sc.location, sc.Format(start))
+ _, err := mkdirIfNotExist(segTemplate, sc.location, sc.Format(start))
if err != nil {
return nil, err
}
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index 07b89a7..2bd94a7 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -228,11 +228,11 @@ type parser interface {
Parse(value string) (time.Time, error)
}
-func loadSections(root string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
+func loadSections(root, prefix string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
var startTimeLst []time.Time
if err := walkDir(
root,
- segPathPrefix,
+ prefix,
func(suffix string) error {
startTime, err := parser.Parse(suffix)
if err != nil {
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index 9e65920..acc50f9 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -350,3 +350,15 @@ func mkdir(format string, a ...interface{}) (path string, err error) {
}
return path, err
}
+
+func mkdirIfNotExist(format string, a ...interface{}) (path string, err error) {
+ path = fmt.Sprintf(format, a...)
+ if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) {
+ if err = os.MkdirAll(path, dirPerm); err != nil {
+ return "", errors.Wrapf(err, "failed to create %s", path)
+ }
+ } else {
+ return "", os.ErrExist
+ }
+ return path, err
+}
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index f7f2882..edd86a5 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -59,7 +59,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
return nil, nil, err
}
if parsedEntity != nil {
- return nil, []tsdb.Entity{parsedEntity}, nil
+ return nil, parsedEntity, nil
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
@@ -168,40 +168,64 @@ func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, ex
return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "index filter parses %v", cond)
}
-func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, tsdb.Entity, error) {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
+func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, []tsdb.Entity, error) {
entityIdx, ok := entityDict[cond.Name]
- if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ {
- return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
+ if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ && cond.Op != modelv1.Condition_BINARY_OP_IN {
+ return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ or IN operation in condition(%v)", cond)
}
switch v := cond.Value.Value.(type) {
case *modelv1.TagValue_Str:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = []byte(v.Str.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return str(v.Str.GetValue()), nil, nil
case *modelv1.TagValue_Id:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = []byte(v.Id.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return id(v.Id.GetValue()), nil, nil
case *modelv1.TagValue_StrArray:
+ if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+ entities := make([]tsdb.Entity, len(v.StrArray.Value))
+ for i, va := range v.StrArray.Value {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
+ parsedEntity[entityIdx] = []byte(va)
+ entities[i] = parsedEntity
+ }
+ return nil, entities, nil
+ }
return &strArrLiteral{
arr: v.StrArray.GetValue(),
}, nil, nil
case *modelv1.TagValue_Int:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = convert.Int64ToBytes(v.Int.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return &int64Literal{
int64: v.Int.GetValue(),
}, nil, nil
case *modelv1.TagValue_IntArray:
+ if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+ entities := make([]tsdb.Entity, len(v.IntArray.Value))
+ for i, va := range v.IntArray.Value {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
+ parsedEntity[entityIdx] = convert.Int64ToBytes(va)
+ entities[i] = parsedEntity
+ }
+ return nil, entities, nil
+ }
return &int64ArrLiteral{
arr: v.IntArray.GetValue(),
}, nil, nil
@@ -214,11 +238,32 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
count := len(input)
result := make(tsdb.Entity, count)
+ anyEntity := func(entities []tsdb.Entity) bool {
+ for _, entity := range entities {
+ for _, entry := range entity {
+ if !bytes.Equal(entry, tsdb.AnyEntry) {
+ return false
+
+ }
+ }
+ }
+ return true
+ }
+ leftAny := anyEntity(left)
+ rightAny := anyEntity(right)
+
mergedEntities := make([]tsdb.Entity, 0, len(left)+len(right))
- mergedEntities = append(mergedEntities, left...)
- mergedEntities = append(mergedEntities, right...)
+
switch op {
case modelv1.LogicalExpression_LOGICAL_OP_AND:
+ if leftAny && !rightAny {
+ return right
+ }
+ if !leftAny && rightAny {
+ return left
+ }
+ mergedEntities = append(mergedEntities, left...)
+ mergedEntities = append(mergedEntities, right...)
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -235,6 +280,14 @@ func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, le
result[i] = entry
}
case modelv1.LogicalExpression_LOGICAL_OP_OR:
+ if leftAny {
+ return left
+ }
+ if rightAny {
+ return right
+ }
+ mergedEntities = append(mergedEntities, left...)
+ mergedEntities = append(mergedEntities, right...)
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -321,7 +374,7 @@ func newAnd(size int) *andNode {
func (an *andNode) merge(list ...posting.List) (posting.List, error) {
var result posting.List
for _, l := range list {
- if _, ok := l.(bypassList); ok {
+ if _, ok := l.(*bypassList); ok {
continue
}
if result == nil {
@@ -366,7 +419,7 @@ func (on *orNode) merge(list ...posting.List) (posting.List, error) {
for _, l := range list {
// If a predicator is not indexed, all predicator are ignored.
// The tagFilter will take up this job to filter this items.
- if _, ok := l.(bypassList); ok {
+ if _, ok := l.(*bypassList); ok {
return bList, nil
}
if result == nil {
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/input/all_only_fields.yaml
similarity index 79%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/input/all_only_fields.yaml
index 7876c40..b3256df 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/input/all_only_fields.yaml
@@ -15,15 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-dataPoints:
- - tagFamilies:
- - name: default
- tags:
- - key: id
- value:
- id:
- value: "1"
- - key: service_id
- value:
- str:
- value: service_1
\ No newline at end of file
+metadata:
+ name: "service_cpm_minute"
+ group: "sw_metric"
+fieldProjection:
+ names: ["total", "value"]
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity.yaml
index 8a16c65..fde1b0c 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity.yaml
@@ -21,7 +21,7 @@ metadata:
tagProjection:
tagFamilies:
- name: "default"
- tags: ["id", "service_id"]
+ tags: ["service_id"]
criteria:
le:
op: "LOGICAL_OP_AND"
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity_in.yaml
similarity index 93%
copy from test/cases/measure/data/input/entity.yaml
copy to test/cases/measure/data/input/entity_in.yaml
index 8a16c65..07c938d 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity_in.yaml
@@ -28,7 +28,7 @@ criteria:
left:
condition:
name: "id"
- op: "BINARY_OP_EQ"
+ op: "BINARY_OP_IN"
value:
- str:
- value: "1"
+ str_array:
+ value: ["1", "2"]
diff --git a/test/cases/measure/data/want/all_only_fields.yaml b/test/cases/measure/data/want/all_only_fields.yaml
new file mode 100644
index 0000000..9e23c73
--- /dev/null
+++ b/test/cases/measure/data/want/all_only_fields.yaml
@@ -0,0 +1,138 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+dataPoints:
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "1"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "10"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "2"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "9"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "3"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "8"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "5"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "11"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "50"
+ - name: value
+ value:
+ int:
+ value: "4"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "2"
+ - key: entity_id
+ value:
+ str:
+ value: entity_1
+ timestamp: "2022-10-17T12:53:45.912Z"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "50"
+ - name: value
+ value:
+ int:
+ value: "12"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "300"
+ - name: value
+ value:
+ int:
+ value: "6"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "300"
+ - name: value
+ value:
+ int:
+ value: "7"
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity.yaml
index 7876c40..8586f55 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity.yaml
@@ -19,10 +19,6 @@ dataPoints:
- tagFamilies:
- name: default
tags:
- - key: id
- value:
- id:
- value: "1"
- key: service_id
value:
str:
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity_in.yaml
similarity index 69%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/want/entity_in.yaml
index 7876c40..5719b1e 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity_in.yaml
@@ -16,14 +16,26 @@
# under the License.
dataPoints:
- - tagFamilies:
- - name: default
- tags:
- - key: id
- value:
- id:
- value: "1"
- - key: service_id
- value:
- str:
- value: service_1
\ No newline at end of file
+- tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "1"
+ - key: service_id
+ value:
+ str:
+ value: service_1
+- tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "2"
+ - key: service_id
+ value:
+ str:
+ value: service_2
+
\ No newline at end of file
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index 97ed283..900bbeb 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -41,6 +41,7 @@ var (
var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("all_only_fields", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by tag", helpers.Args{Input: "tag_filter", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by a integer tag", helpers.Args{Input: "tag_filter_int", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by an unknown tag", helpers.Args{Input: "tag_filter_unknown", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantEmpty: true}),
@@ -53,7 +54,8 @@ var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("limit 3,2", helpers.Args{Input: "limit", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("match a node", helpers.Args{Input: "match_node", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("match nodes", helpers.Args{Input: "match_nodes", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("filter by entity id", helpers.Args{Input: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.FEntry("filter by entity id", helpers.Args{Input: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("filter by several entity ids", helpers.Args{Input: "entity_in", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by entity id and service id", helpers.Args{Input: "entity_service", Want: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("without field", helpers.Args{Input: "no_field", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("invalid logical expression", helpers.Args{Input: "err_invalid_le", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantErr: true}),