You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/01/04 06:25:50 UTC

[skywalking-banyandb] 01/01: Correct the failure of opening existing blocks

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch kv
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit a2ac6f00334027ea2a2a84713f579bbf358f951a
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Jan 4 06:25:07 2023 +0000

    Correct the failure of opening existing blocks
    
    Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
 api/proto/banyandb/measure/v1/query.proto          |   2 +-
 api/proto/banyandb/model/v1/query.proto            |   2 +-
 banyand/tsdb/block_ctrl.go                         |   4 +-
 banyand/tsdb/segment_ctrl.go                       |   4 +-
 banyand/tsdb/shard.go                              |   4 +-
 banyand/tsdb/tsdb.go                               |  12 ++
 pkg/query/logical/index_filter.go                  |  79 ++++++++++--
 .../entity.yaml => input/all_only_fields.yaml}     |  17 +--
 test/cases/measure/data/input/entity.yaml          |   2 +-
 .../data/input/{entity.yaml => entity_in.yaml}     |   6 +-
 test/cases/measure/data/want/all_only_fields.yaml  | 138 +++++++++++++++++++++
 test/cases/measure/data/want/entity.yaml           |   4 -
 .../data/want/{entity.yaml => entity_in.yaml}      |  34 +++--
 test/cases/measure/measure.go                      |   4 +-
 14 files changed, 259 insertions(+), 53 deletions(-)

diff --git a/api/proto/banyandb/measure/v1/query.proto b/api/proto/banyandb/measure/v1/query.proto
index 53df387..8350ba8 100644
--- a/api/proto/banyandb/measure/v1/query.proto
+++ b/api/proto/banyandb/measure/v1/query.proto
@@ -57,7 +57,7 @@ message QueryRequest {
   // tag_families are indexed.
   model.v1.Criteria criteria = 4;
   // tag_projection can be used to select tags of the data points in the response
-  model.v1.TagProjection tag_projection = 5 [(validate.rules).message.required = true];
+  model.v1.TagProjection tag_projection = 5;
   message FieldProjection {
     repeated string names = 1;
   }
diff --git a/api/proto/banyandb/model/v1/query.proto b/api/proto/banyandb/model/v1/query.proto
index cb4a7a1..2fd325b 100644
--- a/api/proto/banyandb/model/v1/query.proto
+++ b/api/proto/banyandb/model/v1/query.proto
@@ -110,7 +110,7 @@ message TagProjection {
     string name = 1;
     repeated string tags = 2;
   }
-  repeated TagFamily tag_families = 1 [(validate.rules).repeated.min_items = 1];
+  repeated TagFamily tag_families = 1;
 }
 
 // TimeRange is a range query for uint64,
diff --git a/banyand/tsdb/block_ctrl.go b/banyand/tsdb/block_ctrl.go
index b1071d3..8d251c4 100644
--- a/banyand/tsdb/block_ctrl.go
+++ b/banyand/tsdb/block_ctrl.go
@@ -226,7 +226,7 @@ func (bc *blockController) closeBlock(ctx context.Context, blockID SectionID) er
 func (bc *blockController) open() error {
 	bc.Lock()
 	defer bc.Unlock()
-	return loadSections(bc.location, bc, bc.blockSize, func(start, end time.Time) error {
+	return loadSections(bc.location, blockPathPrefix, bc, bc.blockSize, func(start, end time.Time) error {
 		_, err := bc.load(start, end, bc.location)
 		return err
 	})
@@ -261,7 +261,7 @@ func (bc *blockController) create(start time.Time) (*block, error) {
 	if end.After(bc.segTimeRange.End) {
 		end = bc.segTimeRange.End
 	}
-	_, err := mkdir(blockTemplate, bc.location, bc.Format(start))
+	_, err := mkdirIfNotExist(blockTemplate, bc.location, bc.Format(start))
 	if err != nil {
 		return nil, err
 	}
diff --git a/banyand/tsdb/segment_ctrl.go b/banyand/tsdb/segment_ctrl.go
index 98653a2..487295f 100644
--- a/banyand/tsdb/segment_ctrl.go
+++ b/banyand/tsdb/segment_ctrl.go
@@ -185,7 +185,7 @@ func (sc *segmentController) Parse(value string) (time.Time, error) {
 func (sc *segmentController) open() error {
 	sc.Lock()
 	defer sc.Unlock()
-	return loadSections(sc.location, sc, sc.segmentSize, func(start, end time.Time) error {
+	return loadSections(sc.location, segPathPrefix, sc, sc.segmentSize, func(start, end time.Time) error {
 		_, err := sc.load(start, end, sc.location)
 		if errors.Is(err, errEndOfSegment) {
 			return nil
@@ -214,7 +214,7 @@ func (sc *segmentController) create(start time.Time) (*segment, error) {
 	} else {
 		end = stdEnd
 	}
-	_, err := mkdir(segTemplate, sc.location, sc.Format(start))
+	_, err := mkdirIfNotExist(segTemplate, sc.location, sc.Format(start))
 	if err != nil {
 		return nil, err
 	}
diff --git a/banyand/tsdb/shard.go b/banyand/tsdb/shard.go
index 07b89a7..2bd94a7 100644
--- a/banyand/tsdb/shard.go
+++ b/banyand/tsdb/shard.go
@@ -228,11 +228,11 @@ type parser interface {
 	Parse(value string) (time.Time, error)
 }
 
-func loadSections(root string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
+func loadSections(root, prefix string, parser parser, intervalRule IntervalRule, loadFn func(start, end time.Time) error) error {
 	var startTimeLst []time.Time
 	if err := walkDir(
 		root,
-		segPathPrefix,
+		prefix,
 		func(suffix string) error {
 			startTime, err := parser.Parse(suffix)
 			if err != nil {
diff --git a/banyand/tsdb/tsdb.go b/banyand/tsdb/tsdb.go
index 9e65920..acc50f9 100644
--- a/banyand/tsdb/tsdb.go
+++ b/banyand/tsdb/tsdb.go
@@ -350,3 +350,15 @@ func mkdir(format string, a ...interface{}) (path string, err error) {
 	}
 	return path, err
 }
+
+func mkdirIfNotExist(format string, a ...interface{}) (path string, err error) {
+	path = fmt.Sprintf(format, a...)
+	if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		if err = os.MkdirAll(path, dirPerm); err != nil {
+			return "", errors.Wrapf(err, "failed to create %s", path)
+		}
+	} else {
+		return "", os.ErrExist
+	}
+	return path, err
+}
diff --git a/pkg/query/logical/index_filter.go b/pkg/query/logical/index_filter.go
index f7f2882..edd86a5 100644
--- a/pkg/query/logical/index_filter.go
+++ b/pkg/query/logical/index_filter.go
@@ -59,7 +59,7 @@ func BuildLocalFilter(criteria *modelv1.Criteria, schema Schema, entityDict map[
 			return nil, nil, err
 		}
 		if parsedEntity != nil {
-			return nil, []tsdb.Entity{parsedEntity}, nil
+			return nil, parsedEntity, nil
 		}
 		if ok, indexRule := schema.IndexDefined(cond.Name); ok {
 			if indexRule.Location == databasev1.IndexRule_LOCATION_GLOBAL {
@@ -168,40 +168,64 @@ func parseCondition(cond *modelv1.Condition, indexRule *databasev1.IndexRule, ex
 	return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "index filter parses %v", cond)
 }
 
-func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, tsdb.Entity, error) {
-	parsedEntity := make(tsdb.Entity, len(entity))
-	copy(parsedEntity, entity)
+func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *modelv1.Condition) (LiteralExpr, []tsdb.Entity, error) {
 	entityIdx, ok := entityDict[cond.Name]
-	if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ {
-		return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ operation in condition(%v)", cond)
+	if ok && cond.Op != modelv1.Condition_BINARY_OP_EQ && cond.Op != modelv1.Condition_BINARY_OP_IN {
+		return nil, nil, errors.WithMessagef(errUnsupportedConditionOp, "tag belongs to the entity only supports EQ or IN operation in condition(%v)", cond)
 	}
 	switch v := cond.Value.Value.(type) {
 	case *modelv1.TagValue_Str:
 		if ok {
+			parsedEntity := make(tsdb.Entity, len(entity))
+			copy(parsedEntity, entity)
 			parsedEntity[entityIdx] = []byte(v.Str.GetValue())
-			return nil, parsedEntity, nil
+			return nil, []tsdb.Entity{parsedEntity}, nil
 		}
 		return str(v.Str.GetValue()), nil, nil
 	case *modelv1.TagValue_Id:
 		if ok {
+			parsedEntity := make(tsdb.Entity, len(entity))
+			copy(parsedEntity, entity)
 			parsedEntity[entityIdx] = []byte(v.Id.GetValue())
-			return nil, parsedEntity, nil
+			return nil, []tsdb.Entity{parsedEntity}, nil
 		}
 		return id(v.Id.GetValue()), nil, nil
 
 	case *modelv1.TagValue_StrArray:
+		if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+			entities := make([]tsdb.Entity, len(v.StrArray.Value))
+			for i, va := range v.StrArray.Value {
+				parsedEntity := make(tsdb.Entity, len(entity))
+				copy(parsedEntity, entity)
+				parsedEntity[entityIdx] = []byte(va)
+				entities[i] = parsedEntity
+			}
+			return nil, entities, nil
+		}
 		return &strArrLiteral{
 			arr: v.StrArray.GetValue(),
 		}, nil, nil
 	case *modelv1.TagValue_Int:
 		if ok {
+			parsedEntity := make(tsdb.Entity, len(entity))
+			copy(parsedEntity, entity)
 			parsedEntity[entityIdx] = convert.Int64ToBytes(v.Int.GetValue())
-			return nil, parsedEntity, nil
+			return nil, []tsdb.Entity{parsedEntity}, nil
 		}
 		return &int64Literal{
 			int64: v.Int.GetValue(),
 		}, nil, nil
 	case *modelv1.TagValue_IntArray:
+		if ok && cond.Op == modelv1.Condition_BINARY_OP_IN {
+			entities := make([]tsdb.Entity, len(v.IntArray.Value))
+			for i, va := range v.IntArray.Value {
+				parsedEntity := make(tsdb.Entity, len(entity))
+				copy(parsedEntity, entity)
+				parsedEntity[entityIdx] = convert.Int64ToBytes(va)
+				entities[i] = parsedEntity
+			}
+			return nil, entities, nil
+		}
 		return &int64ArrLiteral{
 			arr: v.IntArray.GetValue(),
 		}, nil, nil
@@ -214,11 +238,32 @@ func parseExprOrEntity(entityDict map[string]int, entity tsdb.Entity, cond *mode
 func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, left, right []tsdb.Entity) []tsdb.Entity {
 	count := len(input)
 	result := make(tsdb.Entity, count)
+	anyEntity := func(entities []tsdb.Entity) bool {
+		for _, entity := range entities {
+			for _, entry := range entity {
+				if !bytes.Equal(entry, tsdb.AnyEntry) {
+					return false
+
+				}
+			}
+		}
+		return true
+	}
+	leftAny := anyEntity(left)
+	rightAny := anyEntity(right)
+
 	mergedEntities := make([]tsdb.Entity, 0, len(left)+len(right))
-	mergedEntities = append(mergedEntities, left...)
-	mergedEntities = append(mergedEntities, right...)
+
 	switch op {
 	case modelv1.LogicalExpression_LOGICAL_OP_AND:
+		if leftAny && !rightAny {
+			return right
+		}
+		if !leftAny && rightAny {
+			return left
+		}
+		mergedEntities = append(mergedEntities, left...)
+		mergedEntities = append(mergedEntities, right...)
 		for i := 0; i < count; i++ {
 			entry := tsdb.AnyEntry
 			for j := 0; j < len(mergedEntities); j++ {
@@ -235,6 +280,14 @@ func parseEntities(op modelv1.LogicalExpression_LogicalOp, input tsdb.Entity, le
 			result[i] = entry
 		}
 	case modelv1.LogicalExpression_LOGICAL_OP_OR:
+		if leftAny {
+			return left
+		}
+		if rightAny {
+			return right
+		}
+		mergedEntities = append(mergedEntities, left...)
+		mergedEntities = append(mergedEntities, right...)
 		for i := 0; i < count; i++ {
 			entry := tsdb.AnyEntry
 			for j := 0; j < len(mergedEntities); j++ {
@@ -321,7 +374,7 @@ func newAnd(size int) *andNode {
 func (an *andNode) merge(list ...posting.List) (posting.List, error) {
 	var result posting.List
 	for _, l := range list {
-		if _, ok := l.(bypassList); ok {
+		if _, ok := l.(*bypassList); ok {
 			continue
 		}
 		if result == nil {
@@ -366,7 +419,7 @@ func (on *orNode) merge(list ...posting.List) (posting.List, error) {
 	for _, l := range list {
 		// If a predicator is not indexed, all predicator are ignored.
 		// The tagFilter will take up this job to filter this items.
-		if _, ok := l.(bypassList); ok {
+		if _, ok := l.(*bypassList); ok {
 			return bList, nil
 		}
 		if result == nil {
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/input/all_only_fields.yaml
similarity index 79%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/input/all_only_fields.yaml
index 7876c40..b3256df 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/input/all_only_fields.yaml
@@ -15,15 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-dataPoints:
-  - tagFamilies:
-    - name: default
-      tags:
-      - key: id
-        value:
-          id:
-            value: "1"
-      - key: service_id
-        value:
-          str:
-            value: service_1
\ No newline at end of file
+metadata:
+  name: "service_cpm_minute"
+  group: "sw_metric"
+fieldProjection:
+  names: ["total", "value"]
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity.yaml
index 8a16c65..fde1b0c 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity.yaml
@@ -21,7 +21,7 @@ metadata:
 tagProjection:
   tagFamilies:
   - name: "default"
-    tags: ["id", "service_id"]
+    tags: ["service_id"]
 criteria:
   le:
     op: "LOGICAL_OP_AND"
diff --git a/test/cases/measure/data/input/entity.yaml b/test/cases/measure/data/input/entity_in.yaml
similarity index 93%
copy from test/cases/measure/data/input/entity.yaml
copy to test/cases/measure/data/input/entity_in.yaml
index 8a16c65..07c938d 100644
--- a/test/cases/measure/data/input/entity.yaml
+++ b/test/cases/measure/data/input/entity_in.yaml
@@ -28,7 +28,7 @@ criteria:
     left:
       condition:
         name: "id"
-        op: "BINARY_OP_EQ"
+        op: "BINARY_OP_IN"
         value:
-          str:
-            value: "1"
+          str_array:
+            value: ["1", "2"]
diff --git a/test/cases/measure/data/want/all_only_fields.yaml b/test/cases/measure/data/want/all_only_fields.yaml
new file mode 100644
index 0000000..9e23c73
--- /dev/null
+++ b/test/cases/measure/data/want/all_only_fields.yaml
@@ -0,0 +1,138 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+dataPoints:
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "1"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "10"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "2"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "9"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "3"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "8"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "5"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "100"
+  - name: value
+    value:
+      int:
+        value: "11"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "50"
+  - name: value
+    value:
+      int:
+        value: "4"
+  tagFamilies:
+  - name: default
+    tags:
+    - key: id
+      value:
+        id:
+          value: "2"
+    - key: entity_id
+      value:
+        str:
+          value: entity_1
+  timestamp: "2022-10-17T12:53:45.912Z"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "50"
+  - name: value
+    value:
+      int:
+        value: "12"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "300"
+  - name: value
+    value:
+      int:
+        value: "6"
+- fields:
+  - name: total
+    value:
+      int:
+        value: "300"
+  - name: value
+    value:
+      int:
+        value: "7"
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity.yaml
index 7876c40..8586f55 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity.yaml
@@ -19,10 +19,6 @@ dataPoints:
   - tagFamilies:
     - name: default
       tags:
-      - key: id
-        value:
-          id:
-            value: "1"
       - key: service_id
         value:
           str:
diff --git a/test/cases/measure/data/want/entity.yaml b/test/cases/measure/data/want/entity_in.yaml
similarity index 69%
copy from test/cases/measure/data/want/entity.yaml
copy to test/cases/measure/data/want/entity_in.yaml
index 7876c40..5719b1e 100644
--- a/test/cases/measure/data/want/entity.yaml
+++ b/test/cases/measure/data/want/entity_in.yaml
@@ -16,14 +16,26 @@
 # under the License.
 
 dataPoints:
-  - tagFamilies:
-    - name: default
-      tags:
-      - key: id
-        value:
-          id:
-            value: "1"
-      - key: service_id
-        value:
-          str:
-            value: service_1
\ No newline at end of file
+- tagFamilies:
+  - name: default
+    tags:
+    - key: id
+      value:
+        id:
+          value: "1"
+    - key: service_id
+      value:
+        str:
+          value: service_1
+- tagFamilies:
+  - name: default
+    tags:
+    - key: id
+      value:
+        id:
+          value: "2"
+    - key: service_id
+      value:
+        str:
+          value: service_2
+          
\ No newline at end of file
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index 97ed283..900bbeb 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -41,6 +41,7 @@ var (
 
 var _ = g.DescribeTable("Scanning Measures", verify,
 	g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+	g.Entry("all_only_fields", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("filter by tag", helpers.Args{Input: "tag_filter", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("filter by a integer tag", helpers.Args{Input: "tag_filter_int", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("filter by an unknown tag", helpers.Args{Input: "tag_filter_unknown", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantEmpty: true}),
@@ -53,7 +54,8 @@ var _ = g.DescribeTable("Scanning Measures", verify,
 	g.Entry("limit 3,2", helpers.Args{Input: "limit", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("match a node", helpers.Args{Input: "match_node", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("match nodes", helpers.Args{Input: "match_nodes", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
-	g.Entry("filter by entity id", helpers.Args{Input: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+	g.FEntry("filter by entity id", helpers.Args{Input: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
+	g.Entry("filter by several entity ids", helpers.Args{Input: "entity_in", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("filter by entity id and service id", helpers.Args{Input: "entity_service", Want: "entity", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("without field", helpers.Args{Input: "no_field", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
 	g.Entry("invalid logical expression", helpers.Args{Input: "err_invalid_le", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantErr: true}),