You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2021/09/17 22:16:46 UTC
[datasketches-cpp] 01/01: fix issue #236
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch issue_236
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
commit 8ded0a8949a7602d795e9f807c931bc3b29f86ca
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Sep 17 15:16:15 2021 -0700
fix issue #236
---
fi/include/frequent_items_sketch_impl.hpp | 2 +-
kll/include/kll_sketch_impl.hpp | 2 +-
kll/test/kll_sketch_test.cpp | 66 ++++++++++++++++++++++++++++++-
3 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp
index 593aa03..e7db449 100644
--- a/fi/include/frequent_items_sketch_impl.hpp
+++ b/fi/include/frequent_items_sketch_impl.hpp
@@ -350,7 +350,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
check_serial_version(serial_version);
check_family_id(family_id);
check_size(lg_cur_size, lg_max_size);
- ensure_minimum_memory(size, 1ULL << preamble_longs);
+ ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t));
frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
if (!is_empty) {
diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp
index cd309c0..0eb0ae6 100644
--- a/kll/include/kll_sketch_impl.hpp
+++ b/kll/include/kll_sketch_impl.hpp
@@ -575,7 +575,7 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
check_preamble_ints(preamble_ints, flags_byte);
check_serial_version(serial_version);
check_family_id(family_id);
- ensure_minimum_memory(size, 1ULL << preamble_ints);
+ ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t));
const bool is_empty(flags_byte & (1 << flags::IS_EMPTY));
if (is_empty) return kll_sketch<T, C, S, A>(k, allocator);
diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp
index ec49570..c758662 100644
--- a/kll/test/kll_sketch_test.cpp
+++ b/kll/test/kll_sketch_test.cpp
@@ -279,6 +279,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+ REQUIRE(s.tellg() == s.tellp());
REQUIRE(sketch2.is_empty() == sketch.is_empty());
REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -304,7 +305,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
}
- SECTION("serialize deserialize one item") {
+ SECTION("stream serialize deserialize one item") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -324,6 +325,24 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_rank(2) == 1.0);
}
+ SECTION("bytes serialize deserialize one item") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ auto bytes = sketch.serialize();
+ REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+ REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 1);
+ REQUIRE(sketch2.get_num_retained() == 1);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 1.0);
+ REQUIRE(sketch2.get_quantile(0.5) == 1.0);
+ REQUIRE(sketch2.get_rank(1) == 0.0);
+ REQUIRE(sketch2.get_rank(2) == 1.0);
+ }
+
SECTION("deserialize one item v1") {
std::ifstream is;
is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -337,6 +356,42 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch.get_max_value() == 1.0);
}
+ SECTION("stream serialize deserialize three items") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ sketch.update(2.0f);
+ sketch.update(3.0f);
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ sketch.serialize(s);
+ REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+ REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+ REQUIRE(s.tellg() == s.tellp());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 3);
+ REQUIRE(sketch2.get_num_retained() == 3);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 3.0);
+ }
+
+ SECTION("bytes serialize deserialize one item") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ sketch.update(2.0f);
+ sketch.update(3.0f);
+ auto bytes = sketch.serialize();
+ REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+ REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 3);
+ REQUIRE(sketch2.get_num_retained() == 3);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 3.0);
+ }
+
SECTION("stream serialize deserialize many floats") {
kll_float_sketch sketch(200, 0);
const int n = 1000;
@@ -702,6 +757,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
}
+ SECTION("issue #236") {
+ kll_sketch<int8_t> kll;
+ kll.update(1);
+ kll.update(2);
+ kll.update(3);
+ auto blob = kll.serialize();
+ auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
+ }
+
// cleanup
if (test_allocator_total_bytes != 0) {
REQUIRE(test_allocator_total_bytes == 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org