You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by ti...@apache.org on 2022/09/19 07:23:44 UTC
[incubator-kvrocks] branch unstable updated: Move TCL test unit/type/bitmap to Go case (#875)
This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/incubator-kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new d09ab67 Move TCL test unit/type/bitmap to Go case (#875)
d09ab67 is described below
commit d09ab67b71e60383367b1c90441419d949a96fe3
Author: Ruixiang Tan <81...@qq.com>
AuthorDate: Mon Sep 19 15:23:37 2022 +0800
Move TCL test unit/type/bitmap to Go case (#875)
This closes #823.
---
tests/gocase/unit/type/bitmap/bitmap_test.go | 238 +++++++++++++++++++++++++++
tests/tcl/tests/test_helper.tcl | 1 -
tests/tcl/tests/unit/type/bitmap.tcl | 166 -------------------
3 files changed, 238 insertions(+), 167 deletions(-)
diff --git a/tests/gocase/unit/type/bitmap/bitmap_test.go b/tests/gocase/unit/type/bitmap/bitmap_test.go
new file mode 100644
index 0000000..2402ea7
--- /dev/null
+++ b/tests/gocase/unit/type/bitmap/bitmap_test.go
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package bitmap
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/apache/incubator-kvrocks/tests/gocase/util"
+ "github.com/go-redis/redis/v9"
+ "github.com/stretchr/testify/require"
+)
+
+type BITOP int32
+
+const (
+ AND BITOP = 0
+ OR BITOP = 1
+ XOR BITOP = 2
+ NOT BITOP = 3
+)
+
+func Set2SetBit(t *testing.T, rdb *redis.Client, ctx context.Context, key string, bs []byte) {
+ buf := bytes.NewBuffer([]byte{})
+ for _, v := range bs {
+ buf.WriteString(fmt.Sprintf("%08b", v))
+ }
+ for index, value := range buf.String() {
+ require.NoError(t, rdb.SetBit(ctx, key, int64(index), int(value)-int('0')).Err())
+ }
+}
+func GetBitmap(t *testing.T, rdb *redis.Client, ctx context.Context, keys ...string) []string {
+ buf := []string{}
+ for _, key := range keys {
+ cmd := rdb.Get(ctx, key)
+ require.NoError(t, cmd.Err())
+ buf = append(buf, cmd.Val())
+ }
+ return buf
+}
+func SimulateBitOp(op BITOP, values ...[]byte) string {
+ maxlen := 0
+ binaryArray := []string{}
+ for _, value := range values {
+ if maxlen < len(value)*8 {
+ maxlen = len(value) * 8
+ }
+ }
+ for _, value := range values {
+ buf := bytes.NewBuffer([]byte{})
+ for _, v := range value {
+ buf.WriteString(fmt.Sprintf("%08b", v))
+ }
+ tmp := buf.String() + strings.Repeat("0", maxlen-len(buf.String()))
+ binaryArray = append(binaryArray, tmp)
+ }
+ var binarayResult []byte
+ for i := 0; i < maxlen; i++ {
+ x := binaryArray[0][i]
+ if op == NOT {
+ if x == '0' {
+ x = '1'
+ } else {
+ x = '0'
+ }
+ }
+ for j := 1; j < len(binaryArray); j++ {
+ left := int(x - '0')
+ right := int(binaryArray[j][i] - '0')
+ switch op {
+ case AND:
+ left = left & right
+ case XOR:
+ left = left ^ right
+ case OR:
+ left = left | right
+ }
+ if left == 0 {
+ x = '0'
+ } else {
+ x = '1'
+ }
+ }
+ binarayResult = append(binarayResult, x)
+ }
+
+ result := []byte{}
+ for i := 0; i < len(binarayResult); i += 8 {
+ sum := 0
+ for j := 0; j < 8; j++ {
+ sum = sum*2 + int(binarayResult[i+j]-'0')
+ }
+ result = append(result, byte(sum))
+ }
+ return string(result)
+}
+
+func TestBitmap(t *testing.T) {
+ srv := util.StartServer(t, map[string]string{})
+ defer srv.Close()
+ ctx := context.Background()
+ rdb := srv.NewClient()
+ defer func() { require.NoError(t, rdb.Close()) }()
+ t.Run("GET bitmap string after setbit", func(t *testing.T) {
+ require.NoError(t, rdb.SetBit(ctx, "b0", 0, 0).Err())
+ require.NoError(t, rdb.SetBit(ctx, "b1", 35, 0).Err())
+ Set2SetBit(t, rdb, ctx, "b2", []byte("\xac\x81\x32\x5d\xfe"))
+ Set2SetBit(t, rdb, ctx, "b3", []byte("\xff\xff\xff\xff"))
+ require.EqualValues(t, []string{"\x00", "\x00\x00\x00\x00\x00", "\xac\x81\x32\x5d\xfe", "\xff\xff\xff\xff"}, GetBitmap(t, rdb, ctx, "b0", "b1", "b2", "b3"))
+ })
+
+ t.Run("GET bitmap with out of max size", func(t *testing.T) {
+ require.NoError(t, rdb.Do(ctx, "config", "set", "max-bitmap-to-string-mb", 1).Err())
+ require.NoError(t, rdb.SetBit(ctx, "b0", 8388609, 0).Err())
+ util.ErrorRegexp(t, rdb.Get(ctx, "b0").Err(), "ERR Operation aborted: The size of the bitmap .*")
+ })
+
+ t.Run("SETBIT/GETBIT/BITCOUNT/BITPOS boundary check (type bitmap)", func(t *testing.T) {
+ require.NoError(t, rdb.Del(ctx, "b0").Err())
+ maxOffset := 4*1024*1024*1024 - 1
+ util.ErrorRegexp(t, rdb.SetBit(ctx, "b0", int64(maxOffset)+1, 1).Err(), ".*out of range.*")
+ require.NoError(t, rdb.SetBit(ctx, "b0", int64(maxOffset), 1).Err())
+ require.EqualValues(t, 1, rdb.GetBit(ctx, "b0", int64(maxOffset)).Val())
+ require.EqualValues(t, 1, rdb.BitCount(ctx, "b0", &redis.BitCount{Start: 0, End: int64(maxOffset) / 8}).Val())
+ require.EqualValues(t, maxOffset, rdb.BitPos(ctx, "b0", 1).Val())
+ })
+
+ t.Run("BITOP NOT (known string)", func(t *testing.T) {
+ Set2SetBit(t, rdb, ctx, "s", []byte("\xaa\x00\xff\x55"))
+ require.NoError(t, rdb.BitOpNot(ctx, "dest", "s").Err())
+ require.EqualValues(t, []string{"\x55\xff\x00\xaa"}, GetBitmap(t, rdb, ctx, "dest"))
+ })
+
+ t.Run("BITOP where dest and target are the same key", func(t *testing.T) {
+ Set2SetBit(t, rdb, ctx, "s", []byte("\xaa\x00\xff\x55"))
+ require.NoError(t, rdb.BitOpNot(ctx, "s", "s").Err())
+ require.EqualValues(t, []string{"\x55\xff\x00\xaa"}, GetBitmap(t, rdb, ctx, "s"))
+ })
+
+ t.Run("BITOP AND|OR|XOR don't change the string with single input key", func(t *testing.T) {
+ Set2SetBit(t, rdb, ctx, "a", []byte("\x01\x02\xff"))
+ require.NoError(t, rdb.BitOpAnd(ctx, "res1", "a").Err())
+ require.NoError(t, rdb.BitOpOr(ctx, "res2", "a").Err())
+ require.NoError(t, rdb.BitOpXor(ctx, "res3", "a").Err())
+ require.EqualValues(t, []string{"\x01\x02\xff", "\x01\x02\xff", "\x01\x02\xff"}, GetBitmap(t, rdb, ctx, "res1", "res2", "res3"))
+ })
+
+ t.Run("BITOP missing key is considered a stream of zero", func(t *testing.T) {
+ Set2SetBit(t, rdb, ctx, "a", []byte("\x01\x02\xff"))
+ require.NoError(t, rdb.BitOpAnd(ctx, "res1", "no-suck-key", "a").Err())
+ require.NoError(t, rdb.BitOpOr(ctx, "res2", "no-suck-key", "a", "no-suck-key").Err())
+ require.NoError(t, rdb.BitOpXor(ctx, "res3", "no-suck-key", "a").Err())
+ require.EqualValues(t, []string{"\x00\x00\x00", "\x01\x02\xff", "\x01\x02\xff"}, GetBitmap(t, rdb, ctx, "res1", "res2", "res3"))
+ })
+
+ t.Run("BITOP shorter keys are zero-padded to the key with max length", func(t *testing.T) {
+ Set2SetBit(t, rdb, ctx, "a", []byte("\x01\x02\xff\xff"))
+ Set2SetBit(t, rdb, ctx, "b", []byte("\x01\x02\xff"))
+ require.NoError(t, rdb.BitOpAnd(ctx, "res1", "a", "b").Err())
+ require.NoError(t, rdb.BitOpOr(ctx, "res2", "a", "b").Err())
+ require.NoError(t, rdb.BitOpXor(ctx, "res3", "a", "b").Err())
+ require.EqualValues(t, []string{"\x01\x02\xff\x00", "\x01\x02\xff\xff", "\x00\x00\x00\xff"}, GetBitmap(t, rdb, ctx, "res1", "res2", "res3"))
+ })
+
+ for _, op := range []BITOP{AND, OR, XOR} {
+ t.Run("BITOP fuzzing "+strconv.Itoa(int(op)), func(t *testing.T) {
+ for i := 0; i < 10; i++ {
+ require.NoError(t, rdb.FlushAll(ctx).Err())
+ numVec := util.RandomInt(10) + 1
+ var vec [][]byte
+ var veckeys []string
+ for j := 0; j < int(numVec); j++ {
+ str := util.RandString(0, 1000, util.Binary)
+ vec = append(vec, []byte(str))
+ veckeys = append(veckeys, "vector_"+strconv.Itoa(j))
+ Set2SetBit(t, rdb, ctx, "vector_"+strconv.Itoa(j), []byte(str))
+ }
+ switch op {
+ case AND:
+ require.NoError(t, rdb.BitOpAnd(ctx, "target", veckeys...).Err())
+ require.EqualValues(t, SimulateBitOp(AND, vec...), rdb.Get(ctx, "target").Val())
+ case OR:
+ require.NoError(t, rdb.BitOpOr(ctx, "target", veckeys...).Err())
+ require.EqualValues(t, SimulateBitOp(OR, vec...), rdb.Get(ctx, "target").Val())
+ case XOR:
+ require.NoError(t, rdb.BitOpXor(ctx, "target", veckeys...).Err())
+ require.EqualValues(t, SimulateBitOp(XOR, vec...), rdb.Get(ctx, "target").Val())
+ }
+
+ }
+ })
+ }
+
+ t.Run("BITOP NOT fuzzing", func(t *testing.T) {
+ for i := 0; i < 10; i++ {
+ require.NoError(t, rdb.Del(ctx, "str").Err())
+ str := util.RandString(0, 1000, util.Binary)
+ Set2SetBit(t, rdb, ctx, "str", []byte(str))
+ require.NoError(t, rdb.BitOpNot(ctx, "target", "str").Err())
+ require.EqualValues(t, SimulateBitOp(NOT, []byte(str)), rdb.Get(ctx, "target").Val())
+ }
+ })
+
+ t.Run("BITOP with non string source key", func(t *testing.T) {
+ require.NoError(t, rdb.Del(ctx, "c").Err())
+ Set2SetBit(t, rdb, ctx, "a", []byte("\xaa\x00\xff\x55"))
+ Set2SetBit(t, rdb, ctx, "b", []byte("\xaa\x00\xff\x55"))
+ require.NoError(t, rdb.LPush(ctx, "c", "foo").Err())
+ util.ErrorRegexp(t, rdb.BitOpXor(ctx, "dest", "a", "b", "c", "d").Err(), ".*WRONGTYPE.*")
+ })
+
+ t.Run("BITOP with empty string after non empty string (Redis issue #529)", func(t *testing.T) {
+ require.NoError(t, rdb.FlushDB(ctx).Err())
+ Set2SetBit(t, rdb, ctx, "a", []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"))
+ require.EqualValues(t, 32, rdb.BitOpOr(ctx, "x", "a", "b").Val())
+ })
+}
diff --git a/tests/tcl/tests/test_helper.tcl b/tests/tcl/tests/test_helper.tcl
index 895eb72..24fc3ad 100644
--- a/tests/tcl/tests/test_helper.tcl
+++ b/tests/tcl/tests/test_helper.tcl
@@ -38,7 +38,6 @@ set ::all_tests {
unit/type/list
unit/type/zset
unit/type/hash
- unit/type/bitmap
unit/type/stream
unit/multi
unit/expire
diff --git a/tests/tcl/tests/unit/type/bitmap.tcl b/tests/tcl/tests/unit/type/bitmap.tcl
deleted file mode 100644
index 4a73748..0000000
--- a/tests/tcl/tests/unit/type/bitmap.tcl
+++ /dev/null
@@ -1,166 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-proc simulate_bit_op {op args} {
- set maxlen 0
- set j 0
- set count [llength $args]
- foreach a $args {
- binary scan $a b* bits
- set b($j) $bits
- if {[string length $bits] > $maxlen} {
- set maxlen [string length $bits]
- }
- incr j
- }
- for {set j 0} {$j < $count} {incr j} {
- if {[string length $b($j)] < $maxlen} {
- append b($j) [string repeat 0 [expr $maxlen-[string length $b($j)]]]
- }
- }
- set out {}
- for {set x 0} {$x < $maxlen} {incr x} {
- set bit [string range $b(0) $x $x]
- if {$op eq {not}} {set bit [expr {!$bit}]}
- for {set j 1} {$j < $count} {incr j} {
- set bit2 [string range $b($j) $x $x]
- switch $op {
- and {set bit [expr {$bit & $bit2}]}
- or {set bit [expr {$bit | $bit2}]}
- xor {set bit [expr {$bit ^ $bit2}]}
- }
- }
- append out $bit
- }
- binary format b* $out
-}
-
-start_server {tags {"bitmap"}} {
- proc set2setbit {key str} {
- set bitlen [expr {8 * [string length $str]}]
- binary scan $str B$bitlen bit_str
- for {set x 0} {$x < $bitlen} {incr x} {
- r setbit $key $x [string index $bit_str $x]
- }
- }
-
- test {GET bitmap string after setbit} {
- r setbit b0 0 0
- r setbit b1 35 0
- set2setbit b2 "\xac\x81\x32\x5d\xfe"
- set2setbit b3 "\xff\xff\xff\xff"
- list [r get b0] [r get b1] [r get b2] [r get b3]
- } [list "\x00" "\x00\x00\x00\x00\x00" "\xac\x81\x32\x5d\xfe" "\xff\xff\xff\xff"]
-
- test {GET bitmap with out of max size} {
- r config set max-bitmap-to-string-mb 1
- r setbit b0 8388609 0
- catch {r get b0} e
- set e
- } {ERR Operation aborted: The size of the bitmap *}
-
- test "SETBIT/GETBIT/BITCOUNT/BITPOS boundary check (type bitmap)" {
- r del b0
- set max_offset [expr 4*1024*1024*1024-1]
- assert_error "*out of range*" {r setbit b0 [expr $max_offset+1] 1}
- r setbit b0 $max_offset 1
- assert_equal 1 [r getbit b0 $max_offset ]
- assert_equal 1 [r bitcount b0 0 [expr $max_offset / 8] ]
- assert_equal $max_offset [r bitpos b0 1 ]
- }
-
- test {BITOP NOT (known string)} {
- set2setbit s "\xaa\x00\xff\x55"
- r bitop not dest s
- r get dest
- } "\x55\xff\x00\xaa"
-
- test {BITOP where dest and target are the same key} {
- set2setbit s "\xaa\x00\xff\x55"
- r bitop not s s
- r get s
- } "\x55\xff\x00\xaa"
-
- test {BITOP AND|OR|XOR don't change the string with single input key} {
- set2setbit a "\x01\x02\xff"
- r bitop and res1 a
- r bitop or res2 a
- r bitop xor res3 a
- list [r get res1] [r get res2] [r get res3]
- } [list "\x01\x02\xff" "\x01\x02\xff" "\x01\x02\xff"]
-
- test {BITOP missing key is considered a stream of zero} {
- set2setbit a "\x01\x02\xff"
- r bitop and res1 no-suck-key a
- r bitop or res2 no-suck-key a no-such-key
- r bitop xor res3 no-such-key a
- list [r get res1] [r get res2] [r get res3]
- } [list "\x00\x00\x00" "\x01\x02\xff" "\x01\x02\xff"]
-
- test {BITOP shorter keys are zero-padded to the key with max length} {
- set2setbit a "\x01\x02\xff\xff"
- set2setbit b "\x01\x02\xff"
- r bitop and res1 a b
- r bitop or res2 a b
- r bitop xor res3 a b
- list [r get res1] [r get res2] [r get res3]
- } [list "\x01\x02\xff\x00" "\x01\x02\xff\xff" "\x00\x00\x00\xff"]
-
- foreach op {and or xor} {
- test "BITOP $op fuzzing" {
- for {set i 0} {$i < 10} {incr i} {
- r flushall
- set vec {}
- set veckeys {}
- set numvec [expr {[randomInt 10]+1}]
- for {set j 0} {$j < $numvec} {incr j} {
- set str [randstring 0 1000]
- lappend vec $str
- lappend veckeys vector_$j
- set2setbit vector_$j $str
- }
- r bitop $op target {*}$veckeys
- assert_equal [r get target] [simulate_bit_op $op {*}$vec]
- }
- }
- }
-
- test {BITOP NOT fuzzing} {
- for {set i 0} {$i < 10} {incr i} {
- r del str
- set str [randstring 0 1000]
- set2setbit str $str
- r bitop not target str
- assert_equal [r get target] [simulate_bit_op not $str]
- }
- }
-
- test {BITOP with non string source key} {
- r del c
- set2setbit a "\xaa\x00\xff\x55"
- set2setbit b "\xaa\x00\xff\x55"
- r lpush c foo
- catch {r bitop xor dest a b c d} e
- set e
- } {*WRONGTYPE*}
-
- test {BITOP with empty string after non empty string (Redis issue #529)} {
- r flushdb
- set2setbit a "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
- r bitop or x a b
- } {32}
-}
\ No newline at end of file