You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/12/22 19:34:03 UTC
(datasketches-go) branch main updated: Reorganize internal shared code in internal module
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-go.git
The following commit(s) were added to refs/heads/main by this push:
new 9925d8b Reorganize internal shared code in internal module
new b0979ae Merge pull request #6 from freakyzoidberg/reorginize-internal
9925d8b is described below
commit 9925d8b75649f50500a0adb26c88dd55eceb8763
Author: Pierre Lacave <pi...@datadoghq.com>
AuthorDate: Fri Dec 22 09:24:26 2023 +0100
Reorganize internal shared code in internal module
---
Dockerfile => build/Dockerfile | 5 ++-
Dockerfile => build/run-docker-test.sh | 20 +++---------
common/testutils/utils.go | 14 --------
common/utils_slice.go | 29 -----------------
frequencies/longs_sketch.go | 5 +--
frequencies/longs_sketch_serialization_test.go | 28 ++++++++--------
frequencies/longs_sketch_test.go | 6 ++--
frequencies/reverse_purge_long_hash_map.go | 9 +++---
hll/hll_6array.go | 13 ++++----
hll/hll_array.go | 10 +++---
hll/hll_sketch.go | 4 +--
hll/hll_sketch_serialization_test.go | 44 +++++++++++++-------------
hll/hll_utils.go | 3 +-
hll/preamble_utils.go | 9 +++---
hll/relative_error_tables.go | 6 ++--
hll/union.go | 6 ++--
{thetacommon => internal}/quick_select.go | 2 +-
{common => internal}/utils.go | 29 ++++++++++++++++-
{common => internal}/utils_test.go | 2 +-
thetacommon/theta_utils.go | 22 -------------
20 files changed, 107 insertions(+), 159 deletions(-)
diff --git a/Dockerfile b/build/Dockerfile
similarity index 94%
copy from Dockerfile
copy to build/Dockerfile
index a4a18b7..91b06a3 100644
--- a/Dockerfile
+++ b/build/Dockerfile
@@ -20,11 +20,10 @@ ENV CGO_ENABLED=0
WORKDIR /github.com/apache/datasketches-go/
COPY go.mod .
COPY go.sum .
-COPY main.go .
+COPY internal ./internal
COPY common ./common
-COPY thetacommon ./thetacommon
COPY hll ./hll
-
+COPY frequencies ./frequencies
RUN go mod download
RUN go build -v ./...
diff --git a/Dockerfile b/build/run-docker-test.sh
old mode 100644
new mode 100755
similarity index 74%
rename from Dockerfile
rename to build/run-docker-test.sh
index a4a18b7..6c2174b
--- a/Dockerfile
+++ b/build/run-docker-test.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,19 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM golang:alpine
-ENV CGO_ENABLED=0
-
-WORKDIR /github.com/apache/datasketches-go/
-COPY go.mod .
-COPY go.sum .
-COPY main.go .
-COPY common ./common
-COPY thetacommon ./thetacommon
-COPY hll ./hll
-
-
-RUN go mod download
-RUN go build -v ./...
-
-CMD ["go", "test", "-v", "./..."]
+pushd "$(dirname ${BASH_SOURCE:0})"
+trap popd EXIT
+docker build -t datasketch-go-tester -f ./Dockerfile ../ && docker run -t datasketch-go-tester
\ No newline at end of file
diff --git a/common/testutils/utils.go b/common/testutils/utils.go
deleted file mode 100644
index 7f8586a..0000000
--- a/common/testutils/utils.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package testutils
-
-const (
- DSketchTestGenerateGo = "DSKETCH_TEST_GENERATE_GO"
- DSketchTestCrossJava = "DSKETCH_TEST_CROSS_JAVA"
- DSketchTestCrossCpp = "DSKETCH_TEST_CROSS_CPP"
- DSketchTestCrossGo = "DSKETCH_TEST_CROSS_GO"
-)
-
-const (
- JavaPath = "../serialization_test_data/java_generated_files"
- CppPath = "../serialization_test_data/cpp_generated_files"
- GoPath = "../serialization_test_data/go_generated_files"
-)
diff --git a/common/utils_slice.go b/common/utils_slice.go
deleted file mode 100644
index 238fbfc..0000000
--- a/common/utils_slice.go
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package common
-
-// GetShortLE gets a short value from a byte array in little endian format.
-func GetShortLE(array []byte, offset int) int {
- return int(array[offset]&0xFF) | (int(array[offset+1]&0xFF) << 8)
-}
-
-// PutShortLE puts a short value into a byte array in little endian format.
-func PutShortLE(array []byte, offset int, value int) {
- array[offset] = byte(value)
- array[offset+1] = byte(value >> 8)
-}
diff --git a/frequencies/longs_sketch.go b/frequencies/longs_sketch.go
index e28cce1..5543d47 100644
--- a/frequencies/longs_sketch.go
+++ b/frequencies/longs_sketch.go
@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"github.com/apache/datasketches-go/common"
+ "github.com/apache/datasketches-go/internal"
"math/bits"
"strconv"
"strings"
@@ -97,7 +98,7 @@ func NewLongsSketch(lgMaxMapSize int, lgCurMapSize int) (*LongsSketch, error) {
// 0.75 times * maxMapSize. Both the ultimate accuracy and size of this sketch are a
// function of maxMapSize.
func NewLongsSketchWithMaxMapSize(maxMapSize int) (*LongsSketch, error) {
- log2OfInt, err := common.ExactLog2(maxMapSize)
+ log2OfInt, err := internal.ExactLog2(maxMapSize)
if err != nil {
return nil, fmt.Errorf("maxMapSize, %e", err)
}
@@ -287,7 +288,7 @@ func (s *LongsSketch) GetCurrentMapCapacity() int {
//
// maxMapSize is the planned map size to be used when constructing this sketch.
func GetEpsilonLongsSketch(maxMapSize int) (float64, error) {
- if !common.IsPowerOf2(maxMapSize) {
+ if !internal.IsPowerOf2(maxMapSize) {
return 0, errors.New("maxMapSize is not a power of 2")
}
return 3.5 / float64(maxMapSize), nil
diff --git a/frequencies/longs_sketch_serialization_test.go b/frequencies/longs_sketch_serialization_test.go
index c54351d..7fd7cf7 100644
--- a/frequencies/longs_sketch_serialization_test.go
+++ b/frequencies/longs_sketch_serialization_test.go
@@ -19,15 +19,15 @@ package frequencies
import (
"fmt"
- "github.com/apache/datasketches-go/common/testutils"
+ "github.com/apache/datasketches-go/internal"
"github.com/stretchr/testify/assert"
"os"
"testing"
)
func TestGenerateGoBinariesForCompatibilityTestingLongsSketch(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestGenerateGo)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestGenerateGo)
+ if len(os.Getenv(internal.DSketchTestGenerateGo)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestGenerateGo)
}
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
@@ -48,12 +48,12 @@ func TestGenerateGoBinariesForCompatibilityTestingLongsSketch(t *testing.T) {
} else {
assert.Equal(t, sk.GetMaximumError(), int64(0))
}
- err = os.MkdirAll(testutils.GoPath, os.ModePerm)
+ err = os.MkdirAll(internal.GoPath, os.ModePerm)
assert.NoError(t, err)
slc, err := sk.ToSlice()
assert.NoError(t, err)
- err = os.WriteFile(fmt.Sprintf("%s/frequent_long_n%d_go.sk", testutils.GoPath, n), slc, 0644)
+ err = os.WriteFile(fmt.Sprintf("%s/frequent_long_n%d_go.sk", internal.GoPath, n), slc, 0644)
if err != nil {
t.Errorf("err != nil")
}
@@ -61,13 +61,13 @@ func TestGenerateGoBinariesForCompatibilityTestingLongsSketch(t *testing.T) {
}
func TestGoCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossGo)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossGo)
+ if len(os.Getenv(internal.DSketchTestCrossGo)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossGo)
}
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_go.sk", testutils.GoPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_go.sk", internal.GoPath, n))
assert.NoError(t, err)
sketch, err := NewLongsSketchFromSlice(bytes)
if err != nil {
@@ -89,13 +89,13 @@ func TestGoCompat(t *testing.T) {
}
func TestJavaCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossJava)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossJava)
+ if len(os.Getenv(internal.DSketchTestCrossJava)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossJava)
}
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_java.sk", testutils.JavaPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_java.sk", internal.JavaPath, n))
assert.NoError(t, err)
sketch, err := NewLongsSketchFromSlice(bytes)
if err != nil {
@@ -117,13 +117,13 @@ func TestJavaCompat(t *testing.T) {
}
func TestCppCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossCpp)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossCpp)
+ if len(os.Getenv(internal.DSketchTestCrossCpp)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossCpp)
}
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_cpp.sk", testutils.CppPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/frequent_long_n%d_cpp.sk", internal.CppPath, n))
assert.NoError(t, err)
sketch, err := NewLongsSketchFromSlice(bytes)
if err != nil {
diff --git a/frequencies/longs_sketch_test.go b/frequencies/longs_sketch_test.go
index 1d03280..be05cce 100644
--- a/frequencies/longs_sketch_test.go
+++ b/frequencies/longs_sketch_test.go
@@ -20,7 +20,7 @@ package frequencies
import (
"encoding/binary"
"fmt"
- "github.com/apache/datasketches-go/common"
+ "github.com/apache/datasketches-go/internal"
"github.com/stretchr/testify/assert"
"strings"
"testing"
@@ -348,7 +348,7 @@ func TestFreqLongs(t *testing.T) {
}
func newFrequencySketch(eps float64) (*LongsSketch, error) {
- maxMapSize := common.CeilPowerOf2(int(1.0 / (eps * reversePurgeLongHashMapLoadFactor)))
+ maxMapSize := internal.CeilPowerOf2(int(1.0 / (eps * reversePurgeLongHashMapLoadFactor)))
return NewLongsSketchWithMaxMapSize(maxMapSize)
}
@@ -462,7 +462,7 @@ func TestSortItems(t *testing.T) {
numSketches := 1
n := 2222
errorTolerance := 1.0 / 100
- sketchSize := common.CeilPowerOf2(int(1.0 / (errorTolerance * reversePurgeLongHashMapLoadFactor)))
+ sketchSize := internal.CeilPowerOf2(int(1.0 / (errorTolerance * reversePurgeLongHashMapLoadFactor)))
fmt.Printf("sketchSize: %d\n", sketchSize)
sketches := make([]*LongsSketch, numSketches)
diff --git a/frequencies/reverse_purge_long_hash_map.go b/frequencies/reverse_purge_long_hash_map.go
index 3d028b6..ed27f6e 100644
--- a/frequencies/reverse_purge_long_hash_map.go
+++ b/frequencies/reverse_purge_long_hash_map.go
@@ -20,8 +20,7 @@ package frequencies
import (
"errors"
"fmt"
- "github.com/apache/datasketches-go/common"
- "github.com/apache/datasketches-go/thetacommon"
+ "github.com/apache/datasketches-go/internal"
"math/bits"
"strconv"
"strings"
@@ -58,7 +57,7 @@ const (
// The member loadThreshold is then set to the largest value that
// will not overload the hash table.
func newReversePurgeLongHashMap(mapSize int) (*reversePurgeLongHashMap, error) {
- lgLength, err := common.ExactLog2(mapSize)
+ lgLength, err := internal.ExactLog2(mapSize)
if err != nil {
return nil, fmt.Errorf("mapSize: %e", err)
}
@@ -161,7 +160,7 @@ func (r *reversePurgeLongHashMap) purge(sampleSize int) int64 {
i++
}
- val := thetacommon.QuickSelect(samples, 0, numSamples-1, limit/2)
+ val := internal.QuickSelect(samples, 0, numSamples-1, limit/2)
r.adjustAllValuesBy(-1 * val)
r.keepOnlyPositiveCounts()
return val
@@ -368,7 +367,7 @@ func (s *reversePurgeLongHashMap) String() string {
}
func newIterator(keys []int64, values []int64, states []int16, numActive int) *iteratorHashMap {
- stride := int(uint64(float64(len(keys))*common.InverseGolden) | 1)
+ stride := int(uint64(float64(len(keys))*internal.InverseGolden) | 1)
return &iteratorHashMap{
keys_: keys,
values_: values,
diff --git a/hll/hll_6array.go b/hll/hll_6array.go
index 7d7bcae..e761140 100644
--- a/hll/hll_6array.go
+++ b/hll/hll_6array.go
@@ -19,8 +19,7 @@ package hll
import (
"fmt"
-
- "github.com/apache/datasketches-go/common"
+ "github.com/apache/datasketches-go/internal"
)
// hll6ArrayImpl Uses 6 bits per slot in a packed byte array
@@ -128,7 +127,7 @@ func get6Bit(arr []byte, offsetBytes int, slotNo int) int {
startBit := slotNo * 6
shift := startBit & 0x7
byteIdx := (startBit >> 3) + offsetBytes
- return (common.GetShortLE(arr, byteIdx) >> shift) & 0x3F
+ return (internal.GetShortLE(arr, byteIdx) >> shift) & 0x3F
}
func put6Bit(arr []byte, offsetBytes int, slotNo int, newValue int) {
@@ -136,9 +135,9 @@ func put6Bit(arr []byte, offsetBytes int, slotNo int, newValue int) {
shift := startBit & 0x7
byteIdx := (startBit >> 3) + offsetBytes
valShifted := (newValue & 0x3F) << shift
- curMasked := common.GetShortLE(arr, byteIdx) & (^(valMask6 << shift))
+ curMasked := internal.GetShortLE(arr, byteIdx) & (^(valMask6 << shift))
insert := curMasked | valShifted
- common.PutShortLE(arr, byteIdx, insert)
+ internal.PutShortLE(arr, byteIdx, insert)
}
func convertToHll6(srcAbsHllArr hllArray) (hllSketchBase, error) {
@@ -191,7 +190,7 @@ func (h *hll6Iterator) nextValid() bool {
for h.index+1 < h.lengthPairs {
h.index++
h.bitOffset += 6
- tmp := common.GetShortLE(h.hll.hllByteArr, h.bitOffset/8)
+ tmp := internal.GetShortLE(h.hll.hllByteArr, h.bitOffset/8)
h.value = (tmp >> ((h.bitOffset % 8) & 0x7)) & valMask6
if h.value != empty {
return true
@@ -201,7 +200,7 @@ func (h *hll6Iterator) nextValid() bool {
}
func (h *hll6Iterator) getValue() (int, error) {
- tmp := common.GetShortLE(h.hll.hllByteArr, h.bitOffset/8)
+ tmp := internal.GetShortLE(h.hll.hllByteArr, h.bitOffset/8)
shift := (h.bitOffset % 8) & 0x7
return (tmp >> shift) & valMask6, nil
}
diff --git a/hll/hll_array.go b/hll/hll_array.go
index 7348c07..dec51e6 100644
--- a/hll/hll_array.go
+++ b/hll/hll_array.go
@@ -19,7 +19,7 @@ package hll
import (
"fmt"
- "github.com/apache/datasketches-go/common"
+ "github.com/apache/datasketches-go/internal"
)
type hllArray interface {
@@ -274,14 +274,14 @@ func (a *hllArrayImpl) hipAndKxQIncrementalUpdate(oldValue int, newValue int) er
func (a *hllArrayImpl) incrementalUpdateKxQ(oldValue int, newValue int, kxq0 float64, kxq1 float64) error {
//update kxq0 and kxq1; subtract first, then add.
if oldValue < 32 {
- v, err := common.InvPow2(oldValue)
+ v, err := internal.InvPow2(oldValue)
if err != nil {
return err
}
kxq0 -= v
a.kxq0 = kxq0
} else {
- v, err := common.InvPow2(oldValue)
+ v, err := internal.InvPow2(oldValue)
if err != nil {
return err
}
@@ -289,14 +289,14 @@ func (a *hllArrayImpl) incrementalUpdateKxQ(oldValue int, newValue int, kxq0 flo
a.kxq1 = kxq1
}
if newValue < 32 {
- v, err := common.InvPow2(newValue)
+ v, err := internal.InvPow2(newValue)
if err != nil {
return err
}
kxq0 += v
a.kxq0 = kxq0
} else {
- v, err := common.InvPow2(newValue)
+ v, err := internal.InvPow2(newValue)
if err != nil {
return err
}
diff --git a/hll/hll_sketch.go b/hll/hll_sketch.go
index b85e235..8467283 100644
--- a/hll/hll_sketch.go
+++ b/hll/hll_sketch.go
@@ -20,10 +20,10 @@ package hll
import (
"encoding/binary"
"fmt"
+ "github.com/apache/datasketches-go/internal"
"math/bits"
"unsafe"
- "github.com/apache/datasketches-go/thetacommon"
"github.com/twmb/murmur3"
)
@@ -316,5 +316,5 @@ func (h *hllSketchImpl) GetSerializationVersion() int {
}
func (h *hllSketchImpl) hash(bs []byte) (uint64, uint64) {
- return murmur3.SeedSum128(thetacommon.DEFAULT_UPDATE_SEED, thetacommon.DEFAULT_UPDATE_SEED, bs)
+ return murmur3.SeedSum128(internal.DEFAULT_UPDATE_SEED, internal.DEFAULT_UPDATE_SEED, bs)
}
diff --git a/hll/hll_sketch_serialization_test.go b/hll/hll_sketch_serialization_test.go
index 94a5ff1..22658d6 100644
--- a/hll/hll_sketch_serialization_test.go
+++ b/hll/hll_sketch_serialization_test.go
@@ -19,7 +19,7 @@ package hll
import (
"fmt"
- "github.com/apache/datasketches-go/common/testutils"
+ "github.com/apache/datasketches-go/internal"
"os"
"testing"
@@ -28,8 +28,8 @@ import (
// Run me manually for generation
func TestGenerateGoFiles(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestGenerateGo)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestGenerateGo)
+ if len(os.Getenv(internal.DSketchTestGenerateGo)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestGenerateGo)
}
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
@@ -46,35 +46,35 @@ func TestGenerateGoFiles(t *testing.T) {
assert.NoError(t, hll6.UpdateUInt64(uint64(i)))
assert.NoError(t, hll8.UpdateUInt64(uint64(i)))
}
- err = os.MkdirAll(testutils.GoPath, os.ModePerm)
+ err = os.MkdirAll(internal.GoPath, os.ModePerm)
assert.NoError(t, err)
sl4, err := hll4.ToCompactSlice()
assert.NoError(t, err)
- err = os.WriteFile(fmt.Sprintf("%s/hll4_n%d_go.sk", testutils.GoPath, n), sl4, 0644)
+ err = os.WriteFile(fmt.Sprintf("%s/hll4_n%d_go.sk", internal.GoPath, n), sl4, 0644)
assert.NoError(t, err)
sl6, err := hll6.ToCompactSlice()
assert.NoError(t, err)
- err = os.WriteFile(fmt.Sprintf("%s/hll6_n%d_go.sk", testutils.GoPath, n), sl6, 0644)
+ err = os.WriteFile(fmt.Sprintf("%s/hll6_n%d_go.sk", internal.GoPath, n), sl6, 0644)
assert.NoError(t, err)
sl8, err := hll8.ToCompactSlice()
assert.NoError(t, err)
- err = os.WriteFile(fmt.Sprintf("%s/hll8_n%d_go.sk", testutils.GoPath, n), sl8, 0644)
+ err = os.WriteFile(fmt.Sprintf("%s/hll8_n%d_go.sk", internal.GoPath, n), sl8, 0644)
assert.NoError(t, err)
}
}
func TestJavaCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossJava)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossJava)
+ if len(os.Getenv(internal.DSketchTestCrossJava)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossJava)
}
t.Run("Java Hll4", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_java.sk", testutils.JavaPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_java.sk", internal.JavaPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
if err != nil {
@@ -91,7 +91,7 @@ func TestJavaCompat(t *testing.T) {
t.Run("Java Hll6", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_java.sk", testutils.JavaPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_java.sk", internal.JavaPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
@@ -109,7 +109,7 @@ func TestJavaCompat(t *testing.T) {
t.Run("Java Hll8", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_java.sk", testutils.JavaPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_java.sk", internal.JavaPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
if err != nil {
@@ -125,14 +125,14 @@ func TestJavaCompat(t *testing.T) {
}
func TestCppCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossCpp)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossCpp)
+ if len(os.Getenv(internal.DSketchTestCrossCpp)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossCpp)
}
t.Run("Cpp Hll4", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_cpp.sk", testutils.CppPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_cpp.sk", internal.CppPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
if err != nil {
@@ -149,7 +149,7 @@ func TestCppCompat(t *testing.T) {
t.Run("Cpp Hll6", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_cpp.sk", testutils.CppPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_cpp.sk", internal.CppPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
@@ -167,7 +167,7 @@ func TestCppCompat(t *testing.T) {
t.Run("Cpp Hll8", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_cpp.sk", testutils.CppPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_cpp.sk", internal.CppPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
if err != nil {
@@ -183,14 +183,14 @@ func TestCppCompat(t *testing.T) {
}
func TestGoCompat(t *testing.T) {
- if len(os.Getenv(testutils.DSketchTestCrossGo)) == 0 {
- t.Skipf("%s not set", testutils.DSketchTestCrossGo)
+ if len(os.Getenv(internal.DSketchTestCrossGo)) == 0 {
+ t.Skipf("%s not set", internal.DSketchTestCrossGo)
}
t.Run("Go Hll4", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_go.sk", testutils.GoPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll4_n%d_go.sk", internal.GoPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
@@ -208,7 +208,7 @@ func TestGoCompat(t *testing.T) {
t.Run("Go Hll6", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_go.sk", testutils.GoPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll6_n%d_go.sk", internal.GoPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
@@ -226,7 +226,7 @@ func TestGoCompat(t *testing.T) {
t.Run("Go Hll8", func(t *testing.T) {
nArr := []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}
for _, n := range nArr {
- bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_go.sk", testutils.GoPath, n))
+ bytes, err := os.ReadFile(fmt.Sprintf("%s/hll8_n%d_go.sk", internal.GoPath, n))
assert.NoError(t, err)
sketch, err := DeserializeHllSketch(bytes, true)
diff --git a/hll/hll_utils.go b/hll/hll_utils.go
index 419c23d..79fae63 100644
--- a/hll/hll_utils.go
+++ b/hll/hll_utils.go
@@ -19,9 +19,8 @@ package hll
import (
"fmt"
- "math"
-
"github.com/apache/datasketches-go/common"
+ "math"
)
const (
diff --git a/hll/preamble_utils.go b/hll/preamble_utils.go
index cfac779..356c68b 100644
--- a/hll/preamble_utils.go
+++ b/hll/preamble_utils.go
@@ -19,9 +19,8 @@ package hll
import (
"encoding/binary"
+ "github.com/apache/datasketches-go/internal"
"math"
-
- "github.com/apache/datasketches-go/common"
)
const (
@@ -169,16 +168,16 @@ func computeLgArr(byteArr []byte, couponCount int, lgConfigK int) (int, error) {
if curMode == curModeList {
return lgInitListSize, nil
}
- ceilPwr2 := common.CeilPowerOf2(couponCount)
+ ceilPwr2 := internal.CeilPowerOf2(couponCount)
if (resizeDenom * couponCount) > (resizeNumber * ceilPwr2) {
ceilPwr2 <<= 1
}
if curMode == curModeSet {
- v, err := common.ExactLog2(ceilPwr2)
+ v, err := internal.ExactLog2(ceilPwr2)
return max(lgInitSetSize, v), err
}
//only used for HLL4
- v, err := common.ExactLog2(ceilPwr2)
+ v, err := internal.ExactLog2(ceilPwr2)
return max(lgAuxArrInts[lgConfigK], v), err
}
diff --git a/hll/relative_error_tables.go b/hll/relative_error_tables.go
index 5a8acf9..a07dc8a 100644
--- a/hll/relative_error_tables.go
+++ b/hll/relative_error_tables.go
@@ -17,7 +17,9 @@
package hll
-import "github.com/apache/datasketches-go/common"
+import (
+ "github.com/apache/datasketches-go/internal"
+)
var hipLB = //sd 1, 2, 3
[]float64{ //Q(.84134), Q(.97725), Q(.99865) respectively
@@ -76,7 +78,7 @@ var nonHipUB = //sd 1, 2, 3
func getRelErrKLT12(upperBound bool, oooFlag bool, lgK int, stdDev int) float64 {
idx := ((lgK - 4) * 3) + (stdDev - 1)
- sw := (2 * common.BoolToInt(oooFlag)) + common.BoolToInt(upperBound)
+ sw := (2 * internal.BoolToInt(oooFlag)) + internal.BoolToInt(upperBound)
var f float64
switch sw {
case 0:
diff --git a/hll/union.go b/hll/union.go
index bfe5766..80f37b9 100644
--- a/hll/union.go
+++ b/hll/union.go
@@ -19,7 +19,7 @@ package hll
import (
"fmt"
- "github.com/apache/datasketches-go/common"
+ "github.com/apache/datasketches-go/internal"
)
type Union interface {
@@ -303,13 +303,13 @@ func checkRebuildCurMinNumKxQ(sketch HllSketch) error {
}
if v > 0 {
if v < 32 {
- inv, err := common.InvPow2(v)
+ inv, err := internal.InvPow2(v)
if err != nil {
return err
}
kxq0 += inv - 1.0
} else {
- inv, err := common.InvPow2(v)
+ inv, err := internal.InvPow2(v)
if err != nil {
return err
}
diff --git a/thetacommon/quick_select.go b/internal/quick_select.go
similarity index 98%
rename from thetacommon/quick_select.go
rename to internal/quick_select.go
index 14295a1..a1eec9c 100644
--- a/thetacommon/quick_select.go
+++ b/internal/quick_select.go
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package thetacommon
+package internal
func QuickSelect(arr []int64, lo int, hi int, pivot int) int64 {
for hi > 0 {
diff --git a/common/utils.go b/internal/utils.go
similarity index 68%
rename from common/utils.go
rename to internal/utils.go
index 80a8b0d..2ab5181 100644
--- a/common/utils.go
+++ b/internal/utils.go
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package common
+package internal
import (
"fmt"
@@ -28,6 +28,33 @@ const (
InverseGolden = float64(0.6180339887498949025)
)
+const (
+ DEFAULT_UPDATE_SEED = uint64(9001)
+)
+
+const (
+ DSketchTestGenerateGo = "DSKETCH_TEST_GENERATE_GO"
+ DSketchTestCrossJava = "DSKETCH_TEST_CROSS_JAVA"
+ DSketchTestCrossCpp = "DSKETCH_TEST_CROSS_CPP"
+ DSketchTestCrossGo = "DSKETCH_TEST_CROSS_GO"
+)
+
+const (
+ JavaPath = "../serialization_test_data/java_generated_files"
+ CppPath = "../serialization_test_data/cpp_generated_files"
+ GoPath = "../serialization_test_data/go_generated_files"
+)
+
+// GetShortLE gets a short value from a byte array in little endian format.
+func GetShortLE(array []byte, offset int) int {
+ return int(array[offset]&0xFF) | (int(array[offset+1]&0xFF) << 8)
+}
+
+// PutShortLE puts a short value into a byte array in little endian format.
+func PutShortLE(array []byte, offset int, value int) {
+ array[offset] = byte(value)
+ array[offset+1] = byte(value >> 8)
+}
// InvPow2 returns 2^(-e).
func InvPow2(e int) (float64, error) {
diff --git a/common/utils_test.go b/internal/utils_test.go
similarity index 98%
rename from common/utils_test.go
rename to internal/utils_test.go
index db6cb97..ef6a91d 100644
--- a/common/utils_test.go
+++ b/internal/utils_test.go
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package common
+package internal
import (
"github.com/stretchr/testify/assert"
diff --git a/thetacommon/theta_utils.go b/thetacommon/theta_utils.go
deleted file mode 100644
index 403560c..0000000
--- a/thetacommon/theta_utils.go
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package thetacommon
-
-const (
- DEFAULT_UPDATE_SEED = uint64(9001)
-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org