You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/08/07 19:44:42 UTC
[datasketches-cpp] branch java_serde_compat_testing created (now c3f1dc0)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch java_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
at c3f1dc0 added varopt and frequent items sketch tests, fixed IS_EMPTY flag mismatch with Java
This branch includes the following new commits:
new c3f1dc0 added varopt and frequent items sketch tests, fixed IS_EMPTY flag mismatch with Java
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-cpp] 01/01: added varopt and frequent items sketch tests, fixed IS_EMPTY flag mismatch with Java
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch java_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
commit c3f1dc0fd2ea16e9c9400742cd120ab277ba472b
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Aug 7 12:44:31 2023 -0700
added varopt and frequent items sketch tests, fixed IS_EMPTY flag
mismatch with Java
---
fi/include/frequent_items_sketch.hpp | 4 +++-
fi/include/frequent_items_sketch_impl.hpp | 10 ++++++----
fi/test/CMakeLists.txt | 7 +++++++
sampling/test/CMakeLists.txt | 7 +++++++
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/fi/include/frequent_items_sketch.hpp b/fi/include/frequent_items_sketch.hpp
index e3ef94b..0aa9514 100644
--- a/fi/include/frequent_items_sketch.hpp
+++ b/fi/include/frequent_items_sketch.hpp
@@ -298,7 +298,9 @@ private:
static const uint8_t PREAMBLE_LONGS_EMPTY = 1;
static const uint8_t PREAMBLE_LONGS_NONEMPTY = 4;
static constexpr double EPSILON_FACTOR = 3.5;
- enum flags { IS_EMPTY };
+ // due to a mistake different bits were used in C++ and Java to indicate empty sketch
+ // therefore both are set and checked for compatibility with historical binary format
+ enum flags { IS_EMPTY_1 = 0, IS_EMPTY_2 = 2 };
W total_weight;
W offset;
reverse_purge_hash_map<T, W, H, E, A> map;
diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp
index d72f462..acbd2ee 100644
--- a/fi/include/frequent_items_sketch_impl.hpp
+++ b/fi/include/frequent_items_sketch_impl.hpp
@@ -174,7 +174,8 @@ void frequent_items_sketch<T, W, H, E, A>::serialize(std::ostream& os, const Ser
const uint8_t lg_cur_size = map.get_lg_cur_size();
write(os, lg_cur_size);
const uint8_t flags_byte(
- (is_empty() ? 1 << flags::IS_EMPTY : 0)
+ (is_empty() ? 1 << flags::IS_EMPTY_1 : 0)
+ | (is_empty() ? 1 << flags::IS_EMPTY_2 : 0)
);
write(os, flags_byte);
const uint16_t unused16 = 0;
@@ -234,7 +235,8 @@ auto frequent_items_sketch<T, W, H, E, A>::serialize(unsigned header_size_bytes,
const uint8_t lg_cur_size = map.get_lg_cur_size();
ptr += copy_to_mem(lg_cur_size, ptr);
const uint8_t flags_byte(
- (is_empty() ? 1 << flags::IS_EMPTY : 0)
+ (is_empty() ? 1 << flags::IS_EMPTY_1 : 0)
+ | (is_empty() ? 1 << flags::IS_EMPTY_2 : 0)
);
ptr += copy_to_mem(flags_byte, ptr);
ptr += sizeof(uint16_t); // unused
@@ -298,7 +300,7 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
const auto flags_byte = read<uint8_t>(is);
read<uint16_t>(is); // unused
- const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY_1)) | (flags_byte & (1 << flags::IS_EMPTY_2));
check_preamble_longs(preamble_longs, is_empty);
check_serial_version(serial_version);
@@ -352,7 +354,7 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
ptr += copy_from_mem(ptr, flags_byte);
ptr += sizeof(uint16_t); // unused
- const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY_1)) | (flags_byte & (1 << flags::IS_EMPTY_2));
check_preamble_longs(preamble_longs, is_empty);
check_serial_version(serial_version);
diff --git a/fi/test/CMakeLists.txt b/fi/test/CMakeLists.txt
index cd7ffc0..fc8780d 100644
--- a/fi/test/CMakeLists.txt
+++ b/fi/test/CMakeLists.txt
@@ -41,3 +41,10 @@ target_sources(fi_test
frequent_items_sketch_test.cpp
frequent_items_sketch_custom_type_test.cpp
)
+
+if (SERDE_COMPAT)
+target_sources(fi_test
+ PRIVATE
+ frequent_items_sketch_deserialize_from_java_test.cpp
+)
+endif()
diff --git a/sampling/test/CMakeLists.txt b/sampling/test/CMakeLists.txt
index 0a0f930..51764eb 100644
--- a/sampling/test/CMakeLists.txt
+++ b/sampling/test/CMakeLists.txt
@@ -41,3 +41,10 @@ target_sources(sampling_test
var_opt_union_test.cpp
var_opt_allocation_test.cpp
)
+
+if (SERDE_COMPAT)
+target_sources(sampling_test
+ PRIVATE
+ var_opt_sketch_deserialize_from_java_test.cpp
+)
+endif()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org