You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by hu...@apache.org on 2025/05/07 23:25:46 UTC
(kvrocks) branch unstable updated: chore(tests): remove SST load command test and grocksdb (#2942)
This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 2b290b3cf chore(tests): remove SST load command test and grocksdb (#2942)
2b290b3cf is described below
commit 2b290b3cf147189c2ef6aa609b07fca49166f44f
Author: hulk <hu...@gmail.com>
AuthorDate: Thu May 8 07:25:40 2025 +0800
chore(tests): remove SST load command test and grocksdb (#2942)
---
.github/workflows/kvrocks.yaml | 13 +-
tests/gocase/go.mod | 1 -
tests/gocase/go.sum | 2 -
tests/gocase/unit/sst/sst_load_test.go | 609 ---------------------------------
x.py | 17 +-
5 files changed, 6 insertions(+), 636 deletions(-)
diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml
index 0547d88ac..f92262430 100644
--- a/.github/workflows/kvrocks.yaml
+++ b/.github/workflows/kvrocks.yaml
@@ -143,7 +143,6 @@ jobs:
- name: Ubuntu GCC
os: ubuntu-22.04
compiler: gcc
- enable_grocksdb: -tags=enable_grocksdb
- name: SonarCloud with Coverage
os: ubuntu-22.04
compiler: gcc
@@ -327,7 +326,6 @@ jobs:
export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)"
export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)"
export PATH=$PATH:$HOME/local/bin/
- export CGO_ENABLED=1
GOCASE_RUN_ARGS=""
if [[ -n "${{ matrix.with_openssl }}" ]] && [[ "${{ matrix.os }}" == ubuntu* ]]; then
git clone https://github.com/jsha/minica
@@ -338,7 +336,7 @@ jobs:
cp minica.pem tests/gocase/tls/cert/ca.crt
GOCASE_RUN_ARGS="-tlsEnable"
fi
- ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{ matrix.ignore_when_ubsan}} ${{ matrix.enable_grocksdb }}
+ ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{ matrix.ignore_when_ubsan}}
- name: Install redis-py
run: pip3 install redis==5.2.0
@@ -491,7 +489,7 @@ jobs:
- name: Setup openSUSE
if: ${{ startsWith(matrix.image, 'opensuse') }}
run: |
- zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go zlib-devel
+ zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
@@ -503,7 +501,7 @@ jobs:
run: |
dnf install -y epel-release
dnf config-manager --set-enabled powertools
- dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake zlib-devel gcc-toolset-12-libstdc++-devel
+ dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/gcc gcc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
@@ -516,7 +514,7 @@ jobs:
run: |
dnf install -y epel-release
dnf config-manager --set-enabled crb
- dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake zlib-devel
+ dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/gcc gcc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
@@ -535,7 +533,7 @@ jobs:
if: ${{ startsWith(matrix.image, 'alpine') }}
run: |
apk update
- apk add bash cmake curl git python3 wget make gcc g++ autoconf linux-headers py3-pip py3-redis zlib-dev
+ apk add bash cmake curl git python3 wget make gcc g++ autoconf linux-headers py3-pip py3-redis
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Cache redis
@@ -582,7 +580,6 @@ jobs:
- name: Run Go Integration Cases
run: |
export PATH=$PATH:$HOME/local/bin/
- export CGO_ENABLED=1
GOCASE_RUN_ARGS=""
./x.py test go build -parallel 2 $GOCASE_RUN_ARGS
diff --git a/tests/gocase/go.mod b/tests/gocase/go.mod
index 37ca0f112..fd580f129 100644
--- a/tests/gocase/go.mod
+++ b/tests/gocase/go.mod
@@ -3,7 +3,6 @@ module github.com/apache/kvrocks/tests/gocase
go 1.23.0
require (
- github.com/linxGnu/grocksdb v1.9.9
github.com/redis/go-redis/v9 v9.7.3
github.com/shirou/gopsutil/v4 v4.25.2
github.com/stretchr/testify v1.10.0
diff --git a/tests/gocase/go.sum b/tests/gocase/go.sum
index 1dc47e066..62d2d97e2 100644
--- a/tests/gocase/go.sum
+++ b/tests/gocase/go.sum
@@ -15,8 +15,6 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/linxGnu/grocksdb v1.9.9 h1:CzSS/vHLtVIdxdrjvqWR/sm93u/0eB6UcoO14YqObnw=
-github.com/linxGnu/grocksdb v1.9.9/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk=
github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d h1:fjMbDVUGsMQiVZnSQsmouYJvMdwsGiDipOZoN66v844=
github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
diff --git a/tests/gocase/unit/sst/sst_load_test.go b/tests/gocase/unit/sst/sst_load_test.go
deleted file mode 100644
index d2c20bc11..000000000
--- a/tests/gocase/unit/sst/sst_load_test.go
+++ /dev/null
@@ -1,609 +0,0 @@
-//go:build enable_grocksdb
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package sst
-
-import (
- "context"
- "encoding/binary"
- "fmt"
- "log"
- "math/rand"
- "os"
- "path/filepath"
- "sort"
- "testing"
- "time"
-
- "github.com/linxGnu/grocksdb"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/apache/kvrocks/tests/gocase/util"
-)
-
-const (
- DefaultKvrocksNamespace = "__namespace"
- metaDataEncodingMask uint8 = 0x82
- versionCounterBits = 11
-)
-
-type Metadata struct {
- Flags uint8
- Expire uint64
- Version uint64
- Size uint64
-}
-
-type SSTResponse struct {
- filesLoaded int64 `redis:"files_loaded"`
-}
-
-func NewMetadata() *Metadata {
- src := rand.NewSource(time.Now().UnixNano())
- r := rand.New(src)
-
- timestamp := uint64(time.Now().UnixMicro())
- counter := uint64(r.Int63())
- version := (timestamp << versionCounterBits) + (counter % (1 << versionCounterBits))
-
- return &Metadata{
- Flags: metaDataEncodingMask,
- Version: version,
- }
-}
-
-func (m *Metadata) Encode() []byte {
- buf := make([]byte, 25) // 1 + 8 + 8 + 8 bytes
- buf[0] = m.Flags
- binary.BigEndian.PutUint64(buf[1:], m.Expire)
- binary.BigEndian.PutUint64(buf[9:], m.Version)
- binary.BigEndian.PutUint64(buf[17:], m.Size)
- return buf
-}
-
-func encodeInternalKey(namespace, key, field string, version uint64) []byte {
- nsLen := len(namespace)
- keyLen := len(key)
- fieldLen := len(field)
-
- // Pre-calculate total size: 1 byte for ns size + ns + 4 bytes for key size + key + 8 bytes for version + field
- out := make([]byte, 1+nsLen+4+keyLen+8+fieldLen)
-
- out[0] = uint8(nsLen)
- copy(out[1:], namespace)
- binary.BigEndian.PutUint32(out[1+nsLen:], uint32(keyLen))
- copy(out[5+nsLen:], key)
- binary.BigEndian.PutUint64(out[5+nsLen+keyLen:], version)
- copy(out[13+nsLen+keyLen:], field)
-
- return out
-}
-
-func encodeRedisHashKey(namespace, userKey string) []byte {
- totalLen := 1 + len(namespace) + len(userKey)
- buf := make([]byte, totalLen)
- buf[0] = uint8(len(namespace))
- copy(buf[1:], namespace)
- copy(buf[1+len(namespace):], userKey)
- return buf
-}
-
-func createSSTFile(filename string, data map[string]string) error {
- envOpts := grocksdb.NewDefaultEnvOptions()
- sstWriterOpts := grocksdb.NewDefaultOptions()
- sstWriterOpts.SetCompression(grocksdb.CompressionType(2))
- sstWriter := grocksdb.NewSSTFileWriter(envOpts, sstWriterOpts)
- defer sstWriter.Destroy()
-
- err := sstWriter.Open(filename)
- if err != nil {
- return fmt.Errorf("failed to open SST file writer: %v", err)
- }
-
- // Get all keys and sort them
- keys := make([]string, 0, len(data))
- for k := range data {
- keys = append(keys, k)
- }
- sort.Strings(keys)
-
- // Add keys in sorted order
- for _, k := range keys {
- err = sstWriter.Add([]byte(k), []byte(data[k]))
- if err != nil {
- log.Printf("Error adding key %s to SST: %v", k, err)
- }
- }
-
- err = sstWriter.Finish()
- if err != nil {
- return fmt.Errorf("failed to finish SST file: %v", err)
- }
- return nil
-}
-
-func makeTempDir() (string, error) {
- return os.MkdirTemp("", "sst_test_*")
-}
-
-func toInt64(val interface{}) (int64, error) {
- switch v := val.(type) {
- case int64:
- return v, nil
- case int:
- return int64(v), nil
- case float64:
- return int64(v), nil
- default:
- return 0, fmt.Errorf("value is not a number, got %T", val)
- }
-}
-
-func ExtractSSTResponse(result interface{}) (*SSTResponse, error) {
- resultMap, ok := result.(map[interface{}]interface{})
- if !ok {
- return nil, fmt.Errorf("expected map[interface{}]interface{}, got %T", result)
- }
- response := &SSTResponse{}
- for field, target := range map[string]*int64{
- "files_loaded": &response.filesLoaded,
- } {
- if val, ok := resultMap[field]; ok {
- converted, err := toInt64(val)
- if err != nil {
- return nil, fmt.Errorf("%s: %v", field, err)
- }
- *target = converted
- }
- }
- return response, nil
-}
-
-func TestSSTLoad(t *testing.T) {
- configOptions := []util.ConfigOptions{
- {
- Name: "resp3-enabled",
- Options: []string{"yes"},
- ConfigType: util.YesNo,
- },
- }
- configsMatrix, err := util.GenerateConfigsMatrix(configOptions)
- require.NoError(t, err)
- for _, configs := range configsMatrix {
- testSSTLoad(t, configs)
- }
-}
-
-var testSSTLoad = func(t *testing.T, configs util.KvrocksServerConfigs) {
- srv := util.StartServer(t, configs)
- defer srv.Close()
- ctx := context.Background()
- rdb := srv.NewClient()
- defer func() { require.NoError(t, rdb.Close()) }()
-
- t.Run("Test load cmd with no folder", func(t *testing.T) {
- r := rdb.Do(ctx, "sst", "load")
- assert.Error(t, r.Err())
- })
-
- t.Run("Test wrong subcommand", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
- r := rdb.Do(ctx, "sst", "wrong-sub-command", dir)
- assert.Error(t, r.Err())
- })
-
- t.Run("Test wrong load option", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
- r := rdb.Do(ctx, "sst", "load", dir, "wrong-load-option")
- assert.Error(t, r.Err())
- })
-
- t.Run("Test empty folder", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
- r := rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(0), resp.filesLoaded)
- })
-
- t.Run("Test load redis hash keys", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
-
- namespace := DefaultKvrocksNamespace
- data := map[string][]map[string]string{
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- }
- keys := make(map[string]string, len(data))
- metaKeys := make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
- err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys)
- assert.NoError(t, err)
-
- r := rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(2), resp.filesLoaded)
-
- //verify files didn't get moved
- _, err = os.Stat(filepath.Join(dir, "kvrocks_default.sst"))
- assert.NoError(t, err)
- _, err = os.Stat(filepath.Join(dir, "kvrocks_metadata.sst"))
- assert.NoError(t, err)
-
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK, expectedVal := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.NoError(t, val.Err())
- assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK)
- }
- }
- }
- })
-
- t.Run("Test load and update redis hash keys", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
-
- namespace := DefaultKvrocksNamespace
- data := map[string][]map[string]string{
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- }
- keys := make(map[string]string, len(data))
- metaKeys := make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
- err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys)
- assert.NoError(t, err)
-
- r := rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(2), resp.filesLoaded)
-
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK, expectedVal := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.NoError(t, val.Err())
- assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK)
- }
- }
- }
- // update the keys fields via redis interface
- for hashK := range data {
- fields := data[hashK]
- for _, field := range fields {
- for fieldK := range field {
- newVal := "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)
- r := rdb.HSet(ctx, hashK, fieldK, newVal)
- assert.NoError(t, r.Err())
- field[fieldK] = newVal
- }
- }
- }
- // validate the updates made via redis interface
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK, expectedVal := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.NoError(t, val.Err())
- assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK)
- }
- }
- }
- // update the values and sst load
- for hashK := range data {
- fields := data[hashK]
- for _, field := range fields {
- for fieldK := range field {
- newVal := "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)
- field[fieldK] = newVal
- }
- }
- }
-
- keys = make(map[string]string, len(data))
- metaKeys = make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
- err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys)
- assert.NoError(t, err)
-
- r = rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err = ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(2), resp.filesLoaded)
-
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK, expectedVal := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.NoError(t, val.Err())
- assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK)
- }
- }
- }
- })
-
- t.Run("Test load redis hash keys with move option", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
-
- namespace := DefaultKvrocksNamespace
- data := map[string][]map[string]string{
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- }
- keys := make(map[string]string, len(data))
- metaKeys := make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
- err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys)
- assert.NoError(t, err)
-
- r := rdb.Do(ctx, "sst", "load", dir, "movefiles", "yes")
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(2), resp.filesLoaded)
-
- //verify files did get moved
- _, err = os.Stat(filepath.Join(dir, "kvrocks_default.sst"))
- assert.True(t, os.IsNotExist(err))
- _, err = os.Stat(filepath.Join(dir, "kvrocks_metadata.sst"))
- assert.True(t, os.IsNotExist(err))
-
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK, expectedVal := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.NoError(t, val.Err())
- assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK)
- }
- }
- }
- })
-
- t.Run("Test load redis hash keys with no metadata entries", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
-
- namespace := DefaultKvrocksNamespace
- data := map[string][]map[string]string{
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- }
- keys := make(map[string]string, len(data))
- metaKeys := make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
-
- r := rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(1), resp.filesLoaded)
-
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.Error(t, val.Err())
- }
- }
- }
- })
-
- t.Run("Test load redis hash keys with expiration", func(t *testing.T) {
- dir, err := makeTempDir()
- require.NoError(t, err)
- defer os.RemoveAll(dir)
-
- namespace := DefaultKvrocksNamespace
- data := map[string][]map[string]string{
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): {
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)},
- },
- }
- keys := make(map[string]string, len(data))
- metaKeys := make(map[string]string, len(data))
- for hashK := range data {
- meta := NewMetadata()
- meta.Expire = uint64(time.Now().Add(5 * time.Second).UnixMilli())
- fields := data[hashK]
- for _, field := range fields {
- for fieldK, fieldV := range field {
- internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version)
- keys[string(internalKey)] = fieldV
- }
- }
- hashKey := encodeRedisHashKey(namespace, hashK)
- meta.Size++
- metaKeys[string(hashKey)] = string(meta.Encode())
- }
-
- err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys)
- assert.NoError(t, err)
- err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys)
- assert.NoError(t, err)
-
- r := rdb.Do(ctx, "sst", "load", dir)
- assert.NoError(t, r.Err())
- resp, err := ExtractSSTResponse(r.Val())
- assert.NoError(t, err)
- assert.Equal(t, int64(2), resp.filesLoaded)
-
- // verify keys have expiration
- for hashK := range data {
- expireDuration := rdb.ExpireTime(ctx, hashK)
- assert.NotEmpty(t, expireDuration.Val().Milliseconds())
- }
- //verify keys have expired
- time.Sleep(5 * time.Second)
- for hashK, fields := range data {
- for _, field := range fields {
- for fieldK := range field {
- val := rdb.HGet(ctx, hashK, fieldK)
- assert.Error(t, val.Err())
- }
- }
- }
- })
-}
diff --git a/x.py b/x.py
index 787eecade..deb3b4523 100755
--- a/x.py
+++ b/x.py
@@ -223,21 +223,6 @@ def clang_tidy(dir: str, jobs: Optional[int], clang_tidy_path: str, run_clang_ti
run(run_command, *options, *regexes, verbose=True, cwd=basedir)
-def env_with_cgo_flags(build_dir: Path):
- rocksdb_inc = build_dir.joinpath('_deps/rocksdb-src/include')
- rocksdb_lib = build_dir.joinpath('_deps/rocksdb-build')
- zlib_lib = build_dir.joinpath('_deps/zstd-src/lib')
- lz4_lib = build_dir.joinpath('_deps/lz4-src/lib')
- snappy_lib = build_dir.joinpath('_deps/snappy-build')
-
- env = os.environ.copy()
- env.update({
- "CGO_CFLAGS": f"-I{rocksdb_inc} {env.get('CGO_CFLAGS', '')}",
- "CGO_LDFLAGS": f"-L{rocksdb_lib} -L{zlib_lib} -L{lz4_lib} -L{snappy_lib} {env.get('CGO_LDFLAGS', '')}",
- })
- return env
-
-
def golangci_lint(golangci_lint_path: str) -> None:
def get_gopath() -> Tuple[Path, Path]:
go = find_command('go', msg='go is required for testing')
@@ -337,7 +322,7 @@ def test_go(dir: str, cli_path: str, rest: List[str]) -> None:
*rest
]
- run(go, *args, cwd=str(basedir), verbose=True, env=env_with_cgo_flags(Path(dir).absolute()))
+ run(go, *args, cwd=str(basedir), verbose=True)
if __name__ == '__main__':