You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/01/04 07:00:13 UTC
[skywalking-banyandb] branch kv updated (a2ac6f0 -> f1fbd09)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch kv
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
discard a2ac6f0 Correct the failure of opening existing blocks
new f1fbd09 Correct the failure of opening existing blocks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a2ac6f0)
\
N -- N -- N refs/heads/kv (f1fbd09)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
api/proto/banyandb/model/v1/query.proto | 1 -
banyand/tsdb/block_ctrl.go | 3 +--
banyand/tsdb/segment_ctrl.go | 3 +--
banyand/tsdb/tsdb.go | 10 +++++-----
pkg/query/logical/index_filter.go | 1 -
.../measure/data/want/{no_field.yaml => entity_service.yaml} | 0
test/cases/measure/measure.go | 4 ++--
7 files changed, 9 insertions(+), 13 deletions(-)
copy test/cases/measure/data/want/{no_field.yaml => entity_service.yaml} (100%)
[skywalking-banyandb] 01/01: Correct the failure of opening existing blocks
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch kv
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit f1fbd09d4d508cd006a981fa23c8617f5615bfc7
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Jan 4 06:25:07 2023 +0000
Correct the failure of opening existing blocks
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
api/proto/banyandb/measure/v1/query.proto | 2 +-
api/proto/banyandb/model/v1/query.proto | 3 +-
banyand/tsdb/block_ctrl.go | 5 +-
banyand/tsdb/segment_ctrl.go | 5 +-
banyand/tsdb/shard.go | 4 +-
banyand/tsdb/tsdb.go | 12 ++
pkg/query/logical/index_filter.go | 78 ++++++++++--
.../entity.yaml => input/all_only_fields.yaml} | 17 +--
test/cases/measure/data/input/entity.yaml | 2 +-
.../data/input/{entity.yaml => entity_in.yaml} | 6 +-
test/cases/measure/data/want/all_only_fields.yaml | 138 +++++++++++++++++++++
test/cases/measure/data/want/entity.yaml | 4 -
.../data/want/{entity.yaml => entity_in.yaml} | 34 +++--
.../data/want/{entity.yaml => entity_service.yaml} | 0
test/cases/measure/measure.go | 4 +-
15 files changed, 258 insertions(+), 56 deletions(-)
diff --git a/api/proto/banyandb/measure/v1/query.proto b/api/proto/banyandb/measure/v1/query.proto
index 53df387..8350ba8 100644
--- a/api/proto/banyandb/measure/v1/query.proto
+++ b/api/proto/banyandb/measure/v1/query.proto
@@ -57,7 +57,7 @@ message QueryRequest {
// tag_families are indexed.
model.v1.Criteria criteria = 4;
// tag_projection can be used to select tags of the data points in the response
- model.v1.TagProjection tag_projection = 5 [(validate.rules).message.required = true];
+ model.v1.TagProjection tag_projection = 5;
message FieldProjection {
repeated string names = 1;
}
diff --git a/api/proto/banyandb/model/v1/query.proto b/api/proto/banyandb/model/v1/query.proto
index cb4a7a1..2b650ec 100644
--- a/api/proto/banyandb/model/v1/query.proto
+++ b/api/proto/banyandb/model/v1/query.proto
@@ -21,7 +21,6 @@ package banyandb.model.v1;
import "banyandb/model/v1/common.proto";
import "google/protobuf/timestamp.proto";
-import "validate/validate.proto";
option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1";
option java_package = "org.apache.skywalking.banyandb.model.v1";
@@ -110,7 +109,7 @@ message TagProjection {
string name = 1;
repeated string tags = 2;
}
- repeated TagFamily tag_families = 1 [(validate.rules).repeated.min_items = 1];
+ repeated TagFamily tag_families = 1;
}
// TimeRange is a range query for uint64,
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
index b1071d3..d834a51 100644
--- a/banyand/tsdb/block_ctrl.go
+++ b/banyand/tsdb/block_ctrl.go
@@ -226,7 +226,7 @@ func (bc *blockController) closeBlock(ctx context.Context, blockID SectionID) er
func (bc *blockController) open() error {
bc.Lock()
defer bc.Unlock()
- return loadSections(bc.location, bc, bc.blockSize, func(start, end time.Time) error {
+ return loadSections(bc.location, blockPathPrefix, bc, bc.blockSize, func(start, end time.Time) error {
_, err := bc.load(start, end, bc.location)
return err
})
@@ -261,8 +261,7 @@ func (bc *blockController) create(start time.Time) (*block, error) {
if end.After(bc.segTimeRange.End) {
end = bc.segTimeRange.End
}
- _, err := mkdir(blockTemplate, bc.location, bc.Format(start))
- if err != nil {
+ if err := mkdirIfNotExist(blockTemplate, bc.location, bc.Format(start)); err != nil {
return nil, err
}
b, err := bc.load(start, end, bc.location)
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
index 98653a2..4101c01 100644
--- a/banyand/tsdb/segment_ctrl.go
+++ b/banyand/tsdb/segment_ctrl.go
@@ -185,7 +185,7 @@ func (sc *segmentController) Parse(value string) (time.Time, error) {
func (sc *segmentController) open() error {
sc.Lock()
defer sc.Unlock()
- return loadSections(sc.location, sc, sc.segmentSize, func(start, end time.Time) error {
+ return loadSections(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
_, err := sc.load(start, end, sc.location)
if errors.Is(err, errEndOfSegment) {
return nil
@@ -214,8 +214,7 @@ func (sc *segmentController) create(start time.Time) (*segment, error) {
} else {
end = stdEnd
}
- _, err := mkdir(segTemplate, sc.location, sc.Format(start))
- if err != nil {
+ if err := mkdirIfNotExist(segTemplate, sc.location, sc.Format(start)); err != nil {
return nil, err
}
return sc.load(start, end, sc.location)
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index 07b89a7..2bd94a7 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -228,11 +228,11 @@ type parser interface {
Parse(value string) (time.Time, error)
}
-func loadSections(root string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
+func loadSections(root, prefix string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
var startTimeLst []time.Time
if err := walkDir(
root,
- segPathPrefix,
+ prefix,
func(suffix string) error {
startTime, err := parser.Parse(suffix)
if err != nil {
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index 9e65920..65d83d4 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -350,3 +350,15 @@ func mkdir(format string, a ...interface{}) (path string, err error) {
}
return path, err
}
+
+func mkdirIfNotExist(format string, a ...interface{}) (err error) {
+ path := fmt.Sprintf(format, a...)
+ if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) {
+ if err = os.MkdirAll(path, dirPerm); err != nil {
+ return errors.Wrapf(err, "failed to create %s", path)
+ }
+ } else {
+ return os.ErrExist
+ }
+ return err
+}
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index f7f2882..97855a2 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -59,7 +59,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
return nil, nil, err
}
if parsedEntity != nil {
- return nil, []tsdb.Entity{parsedEntity}, nil
+ return nil, parsedEntity, nil
}
if ok, indexRule := schema.IndexDefined(cond.Name); ok {
if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
@@ -168,40 +168,64 @@ func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, ex
return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "index filter parses %v", cond)
}
-func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, tsdb.Entity, error) {
- parsedEntity := make(tsdb.Entity, len(entity))
- copy(parsedEntity, entity)
+func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, []tsdb.Entity, error) {
entityIdx, ok := entityDict[cond.Name]
- if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ {
- return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
+ if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ && cond.Op != modelv1.Condition_BINARY_OP_IN {
+ return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ or IN operation in condition(%v)", cond)
}
switch v := cond.Value.Value.(type) {
case *modelv1.TagValue_Str:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = []byte(v.Str.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return str(v.Str.GetValue()), nil, nil
case *modelv1.TagValue_Id:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = []byte(v.Id.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return id(v.Id.GetValue()), nil, nil
case *modelv1.TagValue_StrArray:
+ if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+ entities := make([]tsdb.Entity, len(v.StrArray.Value))
+ for i, va := range v.StrArray.Value {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
+ parsedEntity[entityIdx] = []byte(va)
+ entities[i] = parsedEntity
+ }
+ return nil, entities, nil
+ }
return &strArrLiteral{
arr: v.StrArray.GetValue(),
}, nil, nil
case *modelv1.TagValue_Int:
if ok {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
parsedEntity[entityIdx] = convert.Int64ToBytes(v.Int.GetValue())
- return nil, parsedEntity, nil
+ return nil, []tsdb.Entity{parsedEntity}, nil
}
return &int64Literal{
int64: v.Int.GetValue(),
}, nil, nil
case *modelv1.TagValue_IntArray:
+ if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+ entities := make([]tsdb.Entity, len(v.IntArray.Value))
+ for i, va := range v.IntArray.Value {
+ parsedEntity := make(tsdb.Entity, len(entity))
+ copy(parsedEntity, entity)
+ parsedEntity[entityIdx] = convert.Int64ToBytes(va)
+ entities[i] = parsedEntity
+ }
+ return nil, entities, nil
+ }
return &int64ArrLiteral{
arr: v.IntArray.GetValue(),
}, nil, nil
@@ -214,11 +238,31 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
count := len(input)
result := make(tsdb.Entity, count)
+ anyEntity := func(entities []tsdb.Entity) bool {
+ for _, entity := range entities {
+ for _, entry := range entity {
+ if !bytes.Equal(entry, tsdb.AnyEntry) {
+ return false
+ }
+ }
+ }
+ return true
+ }
+ leftAny := anyEntity(left)
+ rightAny := anyEntity(right)
+
mergedEntities := make([]tsdb.Entity, 0, len(left)+len(right))
- mergedEntities = append(mergedEntities, left...)
- mergedEntities = append(mergedEntities, right...)
+
switch op {
case modelv1.LogicalExpression_LOGICAL_OP_AND:
+ if leftAny && !rightAny {
+ return right
+ }
+ if !leftAny && rightAny {
+ return left
+ }
+ mergedEntities = append(mergedEntities, left...)
+ mergedEntities = append(mergedEntities, right...)
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -235,6 +279,14 @@ func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, le
result[i] = entry
}
case modelv1.LogicalExpression_LOGICAL_OP_OR:
+ if leftAny {
+ return left
+ }
+ if rightAny {
+ return right
+ }
+ mergedEntities = append(mergedEntities, left...)
+ mergedEntities = append(mergedEntities, right...)
for i := 0; i < count; i++ {
entry := tsdb.AnyEntry
for j := 0; j < len(mergedEntities); j++ {
@@ -321,7 +373,7 @@ func newAnd(size int) *andNode {
func (an *andNode) merge(list ...posting.List) (posting.List, error) {
var result posting.List
for _, l := range list {
- if _, ok := l.(bypassList); ok {
+ if _, ok := l.(*bypassList); ok {
continue
}
if result == nil {
@@ -366,7 +418,7 @@ func (on *orNode) merge(list ...posting.List) (posting.List, error) {
for _, l := range list {
// If a predicator is not indexed, all predicator are ignored.
// The tagFilter will take up this job to filter this items.
- if _, ok := l.(bypassList); ok {
+ if _, ok := l.(*bypassList); ok {
return bList, nil
}
if result == nil {
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/input/all_only_fields.yaml
similarity index 79%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/input/all_only_fields.yaml
index 7876c40..b3256df 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/input/all_only_fields.yaml
@@ -15,15 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-dataPoints:
- - tagFamilies:
- - name: default
- tags:
- - key: id
- value:
- id:
- value: "1"
- - key: service_id
- value:
- str:
- value: service_1
\ No newline at end of file
+metadata:
+ name: "service_cpm_minute"
+ group: "sw_metric"
+fieldProjection:
+ names: ["total", "value"]
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity.yaml
index 8a16c65..fde1b0c 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity.yaml
@@ -21,7 +21,7 @@ metadata:
tagProjection:
tagFamilies:
- name: "default"
- tags: ["id", "service_id"]
+ tags: ["service_id"]
criteria:
le:
op: "LOGICAL_OP_AND"
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity_in.yaml
similarity index 93%
copy from test/cases/measure/data/input/entity.yaml
copy to test/cases/measure/data/input/entity_in.yaml
index 8a16c65..07c938d 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity_in.yaml
@@ -28,7 +28,7 @@ criteria:
left:
condition:
name: "id"
- op: "BINARY_OP_EQ"
+ op: "BINARY_OP_IN"
value:
- str:
- value: "1"
+ str_array:
+ value: ["1", "2"]
diff --git a/test/cases/measure/data/want/all_only_fields.yaml b/test/cases/measure/data/want/all_only_fields.yaml
new file mode 100644
index 0000000..9e23c73
--- /dev/null
+++ b/test/cases/measure/data/want/all_only_fields.yaml
@@ -0,0 +1,138 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+dataPoints:
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "1"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "10"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "2"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "9"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "3"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "8"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "5"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "11"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "50"
+ - name: value
+ value:
+ int:
+ value: "4"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "2"
+ - key: entity_id
+ value:
+ str:
+ value: entity_1
+ timestamp: "2022-10-17T12:53:45.912Z"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "50"
+ - name: value
+ value:
+ int:
+ value: "12"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "300"
+ - name: value
+ value:
+ int:
+ value: "6"
+- fields:
+ - name: total
+ value:
+ int:
+ value: "300"
+ - name: value
+ value:
+ int:
+ value: "7"
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity.yaml
index 7876c40..8586f55 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity.yaml
@@ -19,10 +19,6 @@ dataPoints:
- tagFamilies:
- name: default
tags:
- - key: id
- value:
- id:
- value: "1"
- key: service_id
value:
str:
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity_in.yaml
similarity index 69%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/want/entity_in.yaml
index 7876c40..5719b1e 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity_in.yaml
@@ -16,14 +16,26 @@
# under the License.
dataPoints:
- - tagFamilies:
- - name: default
- tags:
- - key: id
- value:
- id:
- value: "1"
- - key: service_id
- value:
- str:
- value: service_1
\ No newline at end of file
+- tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "1"
+ - key: service_id
+ value:
+ str:
+ value: service_1
+- tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ id:
+ value: "2"
+ - key: service_id
+ value:
+ str:
+ value: service_2
+
\ No newline at end of file
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity_service.yaml
similarity index 100%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/want/entity_service.yaml
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index 97ed283..dfc5cee 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -41,6 +41,7 @@ var (
var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("all_only_fields", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by tag", helpers.Args{Input: "tag_filter", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by a integer tag", helpers.Args{Input: "tag_filter_int", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by an unknown tag", helpers.Args{Input: "tag_filter_unknown", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantEmpty: true}),
@@ -54,7 +55,8 @@ var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("match a node", helpers.Args{Input: "match_node", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("match nodes", helpers.Args{Input: "match_nodes", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("filter by entity id", helpers.Args{Input: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("filter by entity id and service id", helpers.Args{Input: "entity_service", Want: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("filter by several entity ids", helpers.Args{Input: "entity_in", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+ g.Entry("filter by entity id and service id", helpers.Args{Input: "entity_service", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("without field", helpers.Args{Input: "no_field", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("invalid logical expression", helpers.Args{Input: "err_invalid_le", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantErr: true}),
g.Entry("linked or expressions", helpers.Args{Input: "linked_or", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),