You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2022/11/10 16:02:57 UTC
[skywalking-banyandb] 01/01: Sync the block opening
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch open-block
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 11621554730ac4b880585e733df18ea03f697ae3
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Nov 10 16:00:26 2022 +0000
Sync the block opening
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
.github/workflows/ci.yml | 3 +-
.github/workflows/flaky-test.yml | 71 +++++++++++++++++++++++++
Makefile | 5 +-
banyand/tsdb/block.go | 15 ++++--
banyand/tsdb/segment.go | 8 ++-
banyand/tsdb/tsdb_suite_test.go | 2 +-
test/integration/cold_query/query_suite_test.go | 2 +-
test/integration/other/other_suite_test.go | 2 +-
test/integration/query/query_suite_test.go | 2 +-
9 files changed, 95 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 70d0a8a..35da044 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -158,7 +158,8 @@ jobs:
- name: Generate mocks
run: make generate
- name: Test
- run: make test-ci
+ run: |
+ TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out --label-filter !slow" make test-ci
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
result:
diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml
new file mode 100644
index 0000000..7939039
--- /dev/null
+++ b/.github/workflows/flaky-test.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Flaky Test
+
+on:
+ schedule:
+ - cron: '50 * * * *'
+
+env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+ test:
+ name: Test
+ runs-on: ubuntu-20.04
+ strategy:
+ matrix:
+ tz: ["UTC", "Asia/Shanghai", "America/Los_Angeles"]
+ steps:
+ - name: Set timezone
+ run: sudo timedatectl set-timezone ${{ matrix.tz }}
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
+ id: cache-go
+ with:
+ path: |
+ ~/.cache/go-build
+ ~/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+ - name: Cache tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-test-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-test-tool-
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
+ - name: Install Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: 'go.mod'
+ - name: Update dependencies
+ if: steps.cache-go.outputs.cache-hit != 'true'
+ run: GOPROXY=https://proxy.golang.org go mod download
+ - name: Generate mocks
+ run: make generate
+ - name: Test integration and banyand
+ run: TEST_CI_OPTS="--repeat 4 --label-filter (integration&&!slow)||banyand" make test-ci # run tests 4+1 rounds
diff --git a/Makefile b/Makefile
index 259e1da..bc46155 100644
--- a/Makefile
+++ b/Makefile
@@ -29,6 +29,8 @@ include scripts/build/version.mk
PROJECTS := ui banyand bydbctl
+TEST_CI_OPTS ?=
+
##@ Build targets
clean: TARGET=clean
@@ -70,8 +72,7 @@ test-ci: $(GINKGO) ## Run the unit tests in CI
$(GINKGO) --race \
-ldflags \
"-X github.com/apache/skywalking-banyandb/pkg/test/flags.eventuallyTimeout=30s -X github.com/apache/skywalking-banyandb/pkg/test/flags.LogLevel=warn" \
- --cover --covermode atomic --coverprofile=coverage.out \
- --label-filter !slow \
+ $(TEST_CI_OPTS) \
./...
##@ Code quality targets
diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go
index 9a813c0..b7f3158 100644
--- a/banyand/tsdb/block.go
+++ b/banyand/tsdb/block.go
@@ -143,10 +143,19 @@ func (b *block) options(ctx context.Context) {
}
}
-func (b *block) open() (err error) {
- if b.deleted.Load() {
+func (b *block) openSafely() (err error) {
+ if b.deleted.Load() || !b.Closed() {
return nil
}
+ b.lock.Lock()
+ defer b.lock.Unlock()
+ if !b.Closed() {
+ return
+ }
+ return b.open()
+}
+
+func (b *block) open() (err error) {
if b.store, err = kv.OpenTimeSeriesStore(
0,
path.Join(b.path, componentMain),
@@ -194,7 +203,7 @@ func (b *block) delegate(ctx context.Context) (BlockDelegate, error) {
b.lock.Lock()
defer b.lock.Unlock()
if err := b.queue.Push(ctx, blockID, func() error {
- if !b.Closed() {
+ if b.deleted.Load() || !b.Closed() {
return nil
}
return b.open()
diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go
index abfcd28..fbc6fa8 100644
--- a/banyand/tsdb/segment.go
+++ b/banyand/tsdb/segment.go
@@ -196,10 +196,8 @@ func (bc *blockController) Current() (bucket.Reporter, error) {
}
return nil
}(); b != nil {
- if b.Closed() {
- if err := b.open(); err != nil {
- return nil, err
- }
+ if err := b.openSafely(); err != nil {
+ return nil, err
}
return b, nil
}
@@ -390,7 +388,7 @@ func (bc *blockController) create(startTime time.Time) (*block, error) {
if err != nil {
return nil, err
}
- err = b.open()
+ err = b.openSafely()
if err != nil {
return nil, err
}
diff --git a/banyand/tsdb/tsdb_suite_test.go b/banyand/tsdb/tsdb_suite_test.go
index 52e1157..5456e59 100644
--- a/banyand/tsdb/tsdb_suite_test.go
+++ b/banyand/tsdb/tsdb_suite_test.go
@@ -28,7 +28,7 @@ import (
func TestTsdb(t *testing.T) {
RegisterFailHandler(Fail)
- RunSpecs(t, "Tsdb Suite")
+ RunSpecs(t, "Tsdb Suite", Label("banyand"))
}
var _ = BeforeSuite(func() {
diff --git a/test/integration/cold_query/query_suite_test.go b/test/integration/cold_query/query_suite_test.go
index 849a059..2d9285b 100644
--- a/test/integration/cold_query/query_suite_test.go
+++ b/test/integration/cold_query/query_suite_test.go
@@ -40,7 +40,7 @@ import (
func TestIntegrationColdQuery(t *testing.T) {
RegisterFailHandler(Fail)
- RunSpecs(t, "Integration Query Cold Data Suite")
+ RunSpecs(t, "Integration Query Cold Data Suite", Label("integration"))
}
var (
diff --git a/test/integration/other/other_suite_test.go b/test/integration/other/other_suite_test.go
index eaf2474..b073fc8 100644
--- a/test/integration/other/other_suite_test.go
+++ b/test/integration/other/other_suite_test.go
@@ -29,7 +29,7 @@ import (
func TestIntegrationOther(t *testing.T) {
gm.RegisterFailHandler(g.Fail)
- g.RunSpecs(t, "Integration Other Suite")
+ g.RunSpecs(t, "Integration Other Suite", g.Label("integration"))
}
var _ = g.BeforeSuite(func() {
diff --git a/test/integration/query/query_suite_test.go b/test/integration/query/query_suite_test.go
index 69be763..5379dbf 100644
--- a/test/integration/query/query_suite_test.go
+++ b/test/integration/query/query_suite_test.go
@@ -40,7 +40,7 @@ import (
func TestIntegrationQuery(t *testing.T) {
RegisterFailHandler(Fail)
- RunSpecs(t, "Integration Query Suite")
+ RunSpecs(t, "Integration Query Suite", Label("integration"))
}
var (