You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2021/09/17 22:16:45 UTC
[datasketches-cpp] branch issue_236 created (now 8ded0a8)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch issue_236
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git.
at 8ded0a8 fix issue #236
This branch includes the following new commits:
new 8ded0a8 fix issue #236
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-cpp] 01/01: fix issue #236
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch issue_236
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
commit 8ded0a8949a7602d795e9f807c931bc3b29f86ca
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Sep 17 15:16:15 2021 -0700
fix issue #236
---
fi/include/frequent_items_sketch_impl.hpp | 2 +-
kll/include/kll_sketch_impl.hpp | 2 +-
kll/test/kll_sketch_test.cpp | 66 ++++++++++++++++++++++++++++++-
3 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp
index 593aa03..e7db449 100644
--- a/fi/include/frequent_items_sketch_impl.hpp
+++ b/fi/include/frequent_items_sketch_impl.hpp
@@ -350,7 +350,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
check_serial_version(serial_version);
check_family_id(family_id);
check_size(lg_cur_size, lg_max_size);
- ensure_minimum_memory(size, 1ULL << preamble_longs);
+ ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t));
frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
if (!is_empty) {
diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp
index cd309c0..0eb0ae6 100644
--- a/kll/include/kll_sketch_impl.hpp
+++ b/kll/include/kll_sketch_impl.hpp
@@ -575,7 +575,7 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
check_preamble_ints(preamble_ints, flags_byte);
check_serial_version(serial_version);
check_family_id(family_id);
- ensure_minimum_memory(size, 1ULL << preamble_ints);
+ ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t));
const bool is_empty(flags_byte & (1 << flags::IS_EMPTY));
if (is_empty) return kll_sketch<T, C, S, A>(k, allocator);
diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp
index ec49570..c758662 100644
--- a/kll/test/kll_sketch_test.cpp
+++ b/kll/test/kll_sketch_test.cpp
@@ -279,6 +279,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+ REQUIRE(s.tellg() == s.tellp());
REQUIRE(sketch2.is_empty() == sketch.is_empty());
REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -304,7 +305,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
}
- SECTION("serialize deserialize one item") {
+ SECTION("stream serialize deserialize one item") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -324,6 +325,24 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_rank(2) == 1.0);
}
+ SECTION("bytes serialize deserialize one item") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ auto bytes = sketch.serialize();
+ REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+ REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 1);
+ REQUIRE(sketch2.get_num_retained() == 1);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 1.0);
+ REQUIRE(sketch2.get_quantile(0.5) == 1.0);
+ REQUIRE(sketch2.get_rank(1) == 0.0);
+ REQUIRE(sketch2.get_rank(2) == 1.0);
+ }
+
SECTION("deserialize one item v1") {
std::ifstream is;
is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -337,6 +356,42 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch.get_max_value() == 1.0);
}
+ SECTION("stream serialize deserialize three items") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ sketch.update(2.0f);
+ sketch.update(3.0f);
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ sketch.serialize(s);
+ REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+ REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+ REQUIRE(s.tellg() == s.tellp());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 3);
+ REQUIRE(sketch2.get_num_retained() == 3);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 3.0);
+ }
+
+ SECTION("bytes serialize deserialize one item") {
+ kll_float_sketch sketch(200, 0);
+ sketch.update(1.0f);
+ sketch.update(2.0f);
+ sketch.update(3.0f);
+ auto bytes = sketch.serialize();
+ REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+ auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+ REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+ REQUIRE_FALSE(sketch2.is_empty());
+ REQUIRE_FALSE(sketch2.is_estimation_mode());
+ REQUIRE(sketch2.get_n() == 3);
+ REQUIRE(sketch2.get_num_retained() == 3);
+ REQUIRE(sketch2.get_min_value() == 1.0);
+ REQUIRE(sketch2.get_max_value() == 3.0);
+ }
+
SECTION("stream serialize deserialize many floats") {
kll_float_sketch sketch(200, 0);
const int n = 1000;
@@ -702,6 +757,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
}
+ SECTION("issue #236") {
+ kll_sketch<int8_t> kll;
+ kll.update(1);
+ kll.update(2);
+ kll.update(3);
+ auto blob = kll.serialize();
+ auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
+ }
+
// cleanup
if (test_allocator_total_bytes != 0) {
REQUIRE(test_allocator_total_bytes == 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org