You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/12/21 23:16:47 UTC
(datasketches-go) 23/34: Add slice memory corruption test
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-go.git
commit 1d9b6dfb1d13c9c6e96cc10fb772a313c705e78b
Author: Pierre Lacave <pi...@datadoghq.com>
AuthorDate: Wed Dec 20 22:54:58 2023 +0100
Add slice memory corruption test
---
frequencies/long_sketch.go | 12 ++--
frequencies/long_sketch_test.go | 156 ++++++++++++++++++++++++----------------
frequencies/preable_utils.go | 1 +
3 files changed, 102 insertions(+), 67 deletions(-)
diff --git a/frequencies/long_sketch.go b/frequencies/long_sketch.go
index 0fcc9db..996ac6a 100644
--- a/frequencies/long_sketch.go
+++ b/frequencies/long_sketch.go
@@ -164,9 +164,9 @@ func NewLongSketchFromSlice(slc []byte) (*LongSketch, error) {
itemArray[i] = int64(binary.LittleEndian.Uint64(slc[itemsOffset+(i<<3):]))
}
- // Update the sketch
+ // UpdateMany the sketch
for i := 0; i < activeItems; i++ {
- fls.Update(itemArray[i], countArray[i])
+ fls.UpdateMany(itemArray[i], countArray[i])
}
fls.streamWeight = preArr[2] //override streamWeight due to updating
@@ -304,7 +304,11 @@ func (s *LongSketch) isEmpty() bool {
return s.getNumActiveItems() == 0
}
-func (s *LongSketch) Update(item int64, count int64) error {
+func (s *LongSketch) Update(item int64) error {
+ return s.UpdateMany(item, 1)
+}
+
+func (s *LongSketch) UpdateMany(item int64, count int64) error {
if count == 0 {
return nil
}
@@ -341,7 +345,7 @@ func (s *LongSketch) merge(other *LongSketch) (*LongSketch, error) {
streamWt := s.streamWeight + other.streamWeight //capture before merge
iter := other.hashMap.iterator()
for iter.next() {
- err := s.Update(iter.getKey(), iter.getValue())
+ err := s.UpdateMany(iter.getKey(), iter.getValue())
if err != nil {
return nil, err
}
diff --git a/frequencies/long_sketch_test.go b/frequencies/long_sketch_test.go
index 37eea96..d370994 100644
--- a/frequencies/long_sketch_test.go
+++ b/frequencies/long_sketch_test.go
@@ -18,6 +18,7 @@
package frequencies
import (
+ "encoding/binary"
"github.com/stretchr/testify/assert"
"testing"
)
@@ -27,11 +28,11 @@ func TestFrequentItemsStringSerial(t *testing.T) {
assert.NoError(t, err)
sketch2, err := NewLongSketchWithMaxMapSize(128)
assert.NoError(t, err)
- sketch.Update(10, 100)
- sketch.Update(10, 100)
- sketch.Update(15, 3443)
- sketch.Update(1000001, 1010230)
- sketch.Update(1000002, 1010230)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(15, 3443)
+ sketch.UpdateMany(1000001, 1010230)
+ sketch.UpdateMany(1000002, 1010230)
ser, err := sketch.serializeToString()
assert.NoError(t, err)
@@ -43,25 +44,25 @@ func TestFrequentItemsStringSerial(t *testing.T) {
assert.Equal(t, sketch.getMaximumMapCapacity(), newSk0.getMaximumMapCapacity())
assert.Equal(t, sketch.getCurrentMapCapacity(), newSk0.getCurrentMapCapacity())
- sketch2.Update(190, 12902390)
- sketch2.Update(191, 12902390)
- sketch2.Update(192, 12902390)
- sketch2.Update(193, 12902390)
- sketch2.Update(194, 12902390)
- sketch2.Update(195, 12902390)
- sketch2.Update(196, 12902390)
- sketch2.Update(197, 12902390)
- sketch2.Update(198, 12902390)
- sketch2.Update(199, 12902390)
- sketch2.Update(200, 12902390)
- sketch2.Update(201, 12902390)
- sketch2.Update(202, 12902390)
- sketch2.Update(203, 12902390)
- sketch2.Update(204, 12902390)
- sketch2.Update(205, 12902390)
- sketch2.Update(206, 12902390)
- sketch2.Update(207, 12902390)
- sketch2.Update(208, 12902390)
+ sketch2.UpdateMany(190, 12902390)
+ sketch2.UpdateMany(191, 12902390)
+ sketch2.UpdateMany(192, 12902390)
+ sketch2.UpdateMany(193, 12902390)
+ sketch2.UpdateMany(194, 12902390)
+ sketch2.UpdateMany(195, 12902390)
+ sketch2.UpdateMany(196, 12902390)
+ sketch2.UpdateMany(197, 12902390)
+ sketch2.UpdateMany(198, 12902390)
+ sketch2.UpdateMany(199, 12902390)
+ sketch2.UpdateMany(200, 12902390)
+ sketch2.UpdateMany(201, 12902390)
+ sketch2.UpdateMany(202, 12902390)
+ sketch2.UpdateMany(203, 12902390)
+ sketch2.UpdateMany(204, 12902390)
+ sketch2.UpdateMany(205, 12902390)
+ sketch2.UpdateMany(206, 12902390)
+ sketch2.UpdateMany(207, 12902390)
+ sketch2.UpdateMany(208, 12902390)
s2, err := sketch2.serializeToString()
assert.NoError(t, err)
@@ -102,11 +103,11 @@ func TestFrequentItemsByteSerial(t *testing.T) {
sketch2, err := NewLongSketchWithMaxMapSize(128)
assert.NoError(t, err)
- sketch.Update(10, 100)
- sketch.Update(10, 100)
- sketch.Update(15, 3443)
- sketch.Update(1000001, 1010230)
- sketch.Update(1000002, 1010230)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(15, 3443)
+ sketch.UpdateMany(1000001, 1010230)
+ sketch.UpdateMany(1000002, 1010230)
byteArray1, err := sketch.toSlice()
assert.NoError(t, err)
@@ -119,25 +120,25 @@ func TestFrequentItemsByteSerial(t *testing.T) {
assert.Equal(t, sketch.getMaximumMapCapacity(), newSk1.getMaximumMapCapacity())
assert.Equal(t, sketch.getCurrentMapCapacity(), newSk1.getCurrentMapCapacity())
- sketch2.Update(190, 12902390)
- sketch2.Update(191, 12902390)
- sketch2.Update(192, 12902390)
- sketch2.Update(193, 12902390)
- sketch2.Update(194, 12902390)
- sketch2.Update(195, 12902390)
- sketch2.Update(196, 12902390)
- sketch2.Update(197, 12902390)
- sketch2.Update(198, 12902390)
- sketch2.Update(199, 12902390)
- sketch2.Update(200, 12902390)
- sketch2.Update(201, 12902390)
- sketch2.Update(202, 12902390)
- sketch2.Update(203, 12902390)
- sketch2.Update(204, 12902390)
- sketch2.Update(205, 12902390)
- sketch2.Update(206, 12902390)
- sketch2.Update(207, 12902390)
- sketch2.Update(208, 12902390)
+ sketch2.UpdateMany(190, 12902390)
+ sketch2.UpdateMany(191, 12902390)
+ sketch2.UpdateMany(192, 12902390)
+ sketch2.UpdateMany(193, 12902390)
+ sketch2.UpdateMany(194, 12902390)
+ sketch2.UpdateMany(195, 12902390)
+ sketch2.UpdateMany(196, 12902390)
+ sketch2.UpdateMany(197, 12902390)
+ sketch2.UpdateMany(198, 12902390)
+ sketch2.UpdateMany(199, 12902390)
+ sketch2.UpdateMany(200, 12902390)
+ sketch2.UpdateMany(201, 12902390)
+ sketch2.UpdateMany(202, 12902390)
+ sketch2.UpdateMany(203, 12902390)
+ sketch2.UpdateMany(204, 12902390)
+ sketch2.UpdateMany(205, 12902390)
+ sketch2.UpdateMany(206, 12902390)
+ sketch2.UpdateMany(207, 12902390)
+ sketch2.UpdateMany(208, 12902390)
byteArray2, err := sketch2.toSlice()
assert.NoError(t, err)
@@ -171,11 +172,11 @@ func TestFrequentItemsByteSerial(t *testing.T) {
func TestFrequentItemsByteResetAndEmptySerial(t *testing.T) {
sketch, err := NewLongSketchWithMaxMapSize(16)
assert.NoError(t, err)
- sketch.Update(10, 100)
- sketch.Update(10, 100)
- sketch.Update(15, 3443)
- sketch.Update(1000001, 1010230)
- sketch.Update(1000002, 1010230)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(10, 100)
+ sketch.UpdateMany(15, 3443)
+ sketch.UpdateMany(1000001, 1010230)
+ sketch.UpdateMany(1000002, 1010230)
sketch.Reset()
byteArray0, err := sketch.toSlice()
@@ -195,11 +196,11 @@ func TestFreqLongSliceSerDe(t *testing.T) {
minSize := 1 << _LG_MIN_MAP_SIZE
sk1, err := NewLongSketchWithMaxMapSize(minSize)
assert.NoError(t, err)
- sk1.Update(10, 100)
- sk1.Update(10, 100)
- sk1.Update(15, 3443)
- sk1.Update(1000001, 1010230)
- sk1.Update(1000002, 1010230)
+ sk1.UpdateMany(10, 100)
+ sk1.UpdateMany(10, 100)
+ sk1.UpdateMany(15, 3443)
+ sk1.UpdateMany(1000001, 1010230)
+ sk1.UpdateMany(1000002, 1010230)
byteArray0, err := sk1.toSlice()
assert.NoError(t, err)
@@ -213,11 +214,11 @@ func TestFreqLongStringSerDe(t *testing.T) {
minSize := 1 << _LG_MIN_MAP_SIZE
sk1, err := NewLongSketchWithMaxMapSize(minSize)
assert.NoError(t, err)
- sk1.Update(10, 100)
- sk1.Update(10, 100)
- sk1.Update(15, 3443)
- sk1.Update(1000001, 1010230)
- sk1.Update(1000002, 1010230)
+ sk1.UpdateMany(10, 100)
+ sk1.UpdateMany(10, 100)
+ sk1.UpdateMany(15, 3443)
+ sk1.UpdateMany(1000001, 1010230)
+ sk1.UpdateMany(1000002, 1010230)
str1, err := sk1.serializeToString()
assert.NoError(t, err)
@@ -261,3 +262,32 @@ func checkEquality(t *testing.T, sk1, sk2 *LongSketch) {
assert.Equal(t, s1, s2)
}
}
+
+func TestFreqLongSliceSerDeError(t *testing.T) {
+ minSize := 1 << _LG_MIN_MAP_SIZE
+ sk1, err := NewLongSketchWithMaxMapSize(minSize)
+ assert.NoError(t, err)
+ sk1.Update(1)
+
+ byteArray0, err := sk1.toSlice()
+ pre0 := binary.LittleEndian.Uint64(byteArray0)
+
+ tryBadMem(t, byteArray0, _PREAMBLE_LONGS_BYTE, 2) //Corrupt
+ binary.LittleEndian.PutUint64(byteArray0, pre0)
+
+ tryBadMem(t, byteArray0, _SER_VER_BYTE, 2) //Corrupt
+ binary.LittleEndian.PutUint64(byteArray0, pre0)
+
+ tryBadMem(t, byteArray0, _FAMILY_BYTE, 2) //Corrupt
+ binary.LittleEndian.PutUint64(byteArray0, pre0)
+
+ tryBadMem(t, byteArray0, _FLAGS_BYTE, 4) //Corrupt
+ binary.LittleEndian.PutUint64(byteArray0, pre0)
+
+}
+
+func tryBadMem(t *testing.T, mem []byte, byteOffset, byteValue int) {
+ binary.LittleEndian.PutUint64(mem[byteOffset:], uint64(byteValue))
+ _, err := NewLongSketchFromSlice(mem)
+ assert.Error(t, err)
+}
diff --git a/frequencies/preable_utils.go b/frequencies/preable_utils.go
index 7180832..ca3854c 100644
--- a/frequencies/preable_utils.go
+++ b/frequencies/preable_utils.go
@@ -25,6 +25,7 @@ import (
const (
// ###### DO NOT MESS WITH THIS FROM HERE ...
// Preamble byte Addresses
+ _PREAMBLE_LONGS_BYTE = 0
_SER_VER_BYTE = 1
_FAMILY_BYTE = 2
_LG_MAX_MAP_SIZE_BYTE = 3
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org