You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by "jmalkin (via GitHub)" <gi...@apache.org> on 2023/04/18 00:45:20 UTC
[GitHub] [datasketches-cpp] jmalkin opened a new pull request, #362: Density serialization
jmalkin opened a new pull request, #362:
URL: https://github.com/apache/datasketches-cpp/pull/362
Adds density serialization
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] jmalkin merged pull request #362: Density serialization
Posted by "jmalkin (via GitHub)" <gi...@apache.org>.
jmalkin merged PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170719890
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +162,259 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
Review Comment:
shouldn't this be a (const) ref?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170720031
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +162,259 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
Review Comment:
and here const ref perhaps?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170592827
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
+ ptr += copy_to_mem(pt.data(), ptr, pt_size);
+ }
+ }
+
+ if (ptr != end_ptr)
+ throw std::runtime_error("Actual output size does not equal expected output size");
+
+ return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+ const auto preamble_ints = read<uint8_t>(is);
+ const auto serial_version = read<uint8_t>(is);
+ const auto family_id = read<uint8_t>(is);
+ const auto flags_byte = read<uint8_t>(is);
+ const auto k = read<uint16_t>(is);
+ read<uint16_t>(is); // unused
+ const auto dim = read<uint32_t>(is);
+
+ //check_k(k); // do we have constraints?
+ check_serial_version(serial_version); // a little redundant with the header check
+ check_family_id(family_id);
+ check_header_validity(preamble_ints, flags_byte, serial_version);
+
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+ if (is_empty) {
+ return density_sketch(k, dim, kernel, allocator);
+ }
+
+ const auto num_retained = read<uint32_t>(is);
+ const auto n = read<uint64_t>(is);
+
+ // levels arrays
+ size_t pt_size = sizeof(T) * dim;
+ Levels levels(allocator);
+ int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+ while (num_to_read > 0) {
+ const auto level_size = read<uint32_t>(is);
+ Level lvl(allocator);
+ lvl.reserve(level_size);
+ for (uint32_t i = 0; i < level_size; ++i) {
+ Vector pt(dim, 0, allocator);
+ read(is, pt.data(), pt_size);
+ lvl.push_back(pt);
+ }
+ levels.push_back(lvl);
+ num_to_read -= lvl.size();
+ }
+
+ if (num_to_read != 0)
+ throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+ return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+ ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* end_ptr = static_cast<const char*>(bytes) + size;
+ uint8_t preamble_ints;
+ ptr += copy_from_mem(ptr, preamble_ints);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t family_id;
+ ptr += copy_from_mem(ptr, family_id);
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t k;
+ ptr += copy_from_mem(ptr, k);
+ uint16_t unused;
+ ptr += copy_from_mem(ptr, unused);
+ uint32_t dim;
+ ptr += copy_from_mem(ptr, dim);
+
+ //check_k(k);
Review Comment:
I would agree to that, if there are no other practical limits
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170730046
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +162,259 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
Review Comment:
const ref
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170478518
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
+ ptr += copy_to_mem(pt.data(), ptr, pt_size);
+ }
+ }
+
+ if (ptr != end_ptr)
+ throw std::runtime_error("Actual output size does not equal expected output size");
+
+ return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+ const auto preamble_ints = read<uint8_t>(is);
+ const auto serial_version = read<uint8_t>(is);
+ const auto family_id = read<uint8_t>(is);
+ const auto flags_byte = read<uint8_t>(is);
+ const auto k = read<uint16_t>(is);
+ read<uint16_t>(is); // unused
+ const auto dim = read<uint32_t>(is);
+
+ //check_k(k); // do we have constraints?
+ check_serial_version(serial_version); // a little redundant with the header check
+ check_family_id(family_id);
+ check_header_validity(preamble_ints, flags_byte, serial_version);
+
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+ if (is_empty) {
+ return density_sketch(k, dim, kernel, allocator);
+ }
+
+ const auto num_retained = read<uint32_t>(is);
+ const auto n = read<uint64_t>(is);
+
+ // levels arrays
+ size_t pt_size = sizeof(T) * dim;
+ Levels levels(allocator);
+ int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+ while (num_to_read > 0) {
+ const auto level_size = read<uint32_t>(is);
+ Level lvl(allocator);
+ lvl.reserve(level_size);
+ for (uint32_t i = 0; i < level_size; ++i) {
+ Vector pt(dim, 0, allocator);
+ read(is, pt.data(), pt_size);
+ lvl.push_back(pt);
+ }
+ levels.push_back(lvl);
+ num_to_read -= lvl.size();
+ }
+
+ if (num_to_read != 0)
+ throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+ return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+ ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* end_ptr = static_cast<const char*>(bytes) + size;
+ uint8_t preamble_ints;
+ ptr += copy_from_mem(ptr, preamble_ints);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t family_id;
+ ptr += copy_from_mem(ptr, family_id);
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t k;
+ ptr += copy_from_mem(ptr, k);
+ uint16_t unused;
+ ptr += copy_from_mem(ptr, unused);
+ uint32_t dim;
+ ptr += copy_from_mem(ptr, dim);
+
+ //check_k(k);
Review Comment:
do we need to check K here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] jmalkin commented on a diff in pull request #362: Density serialization
Posted by "jmalkin (via GitHub)" <gi...@apache.org>.
jmalkin commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170536446
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
+ ptr += copy_to_mem(pt.data(), ptr, pt_size);
+ }
+ }
+
+ if (ptr != end_ptr)
+ throw std::runtime_error("Actual output size does not equal expected output size");
+
+ return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+ const auto preamble_ints = read<uint8_t>(is);
+ const auto serial_version = read<uint8_t>(is);
+ const auto family_id = read<uint8_t>(is);
+ const auto flags_byte = read<uint8_t>(is);
+ const auto k = read<uint16_t>(is);
+ read<uint16_t>(is); // unused
+ const auto dim = read<uint32_t>(is);
+
+ //check_k(k); // do we have constraints?
+ check_serial_version(serial_version); // a little redundant with the header check
+ check_family_id(family_id);
+ check_header_validity(preamble_ints, flags_byte, serial_version);
+
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+ if (is_empty) {
+ return density_sketch(k, dim, kernel, allocator);
+ }
+
+ const auto num_retained = read<uint32_t>(is);
+ const auto n = read<uint64_t>(is);
+
+ // levels arrays
+ size_t pt_size = sizeof(T) * dim;
+ Levels levels(allocator);
+ int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+ while (num_to_read > 0) {
+ const auto level_size = read<uint32_t>(is);
+ Level lvl(allocator);
+ lvl.reserve(level_size);
+ for (uint32_t i = 0; i < level_size; ++i) {
+ Vector pt(dim, 0, allocator);
+ read(is, pt.data(), pt_size);
+ lvl.push_back(pt);
+ }
+ levels.push_back(lvl);
+ num_to_read -= lvl.size();
+ }
+
+ if (num_to_read != 0)
+ throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+ return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+ ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* end_ptr = static_cast<const char*>(bytes) + size;
+ uint8_t preamble_ints;
+ ptr += copy_from_mem(ptr, preamble_ints);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t family_id;
+ ptr += copy_from_mem(ptr, family_id);
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t k;
+ ptr += copy_from_mem(ptr, k);
+ uint16_t unused;
+ ptr += copy_from_mem(ptr, unused);
+ uint32_t dim;
+ ptr += copy_from_mem(ptr, dim);
+
+ //check_k(k);
Review Comment:
My top choice would be to check. I'm not sure of a practical minimum though. Fairly sure 1 is not a good idea, and 0 should not be allowed. So maybe uncomment, check >1, and also add to the constructor?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170534493
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
+ ptr += copy_to_mem(pt.data(), ptr, pt_size);
+ }
+ }
+
+ if (ptr != end_ptr)
+ throw std::runtime_error("Actual output size does not equal expected output size");
+
+ return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+ const auto preamble_ints = read<uint8_t>(is);
+ const auto serial_version = read<uint8_t>(is);
+ const auto family_id = read<uint8_t>(is);
+ const auto flags_byte = read<uint8_t>(is);
+ const auto k = read<uint16_t>(is);
+ read<uint16_t>(is); // unused
+ const auto dim = read<uint32_t>(is);
+
+ //check_k(k); // do we have constraints?
+ check_serial_version(serial_version); // a little redundant with the header check
+ check_family_id(family_id);
+ check_header_validity(preamble_ints, flags_byte, serial_version);
+
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+ if (is_empty) {
+ return density_sketch(k, dim, kernel, allocator);
+ }
+
+ const auto num_retained = read<uint32_t>(is);
+ const auto n = read<uint64_t>(is);
+
+ // levels arrays
+ size_t pt_size = sizeof(T) * dim;
+ Levels levels(allocator);
+ int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+ while (num_to_read > 0) {
+ const auto level_size = read<uint32_t>(is);
+ Level lvl(allocator);
+ lvl.reserve(level_size);
+ for (uint32_t i = 0; i < level_size; ++i) {
+ Vector pt(dim, 0, allocator);
+ read(is, pt.data(), pt_size);
+ lvl.push_back(pt);
+ }
+ levels.push_back(lvl);
+ num_to_read -= lvl.size();
+ }
+
+ if (num_to_read != 0)
+ throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+ return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+ ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* end_ptr = static_cast<const char*>(bytes) + size;
+ uint8_t preamble_ints;
+ ptr += copy_from_mem(ptr, preamble_ints);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t family_id;
+ ptr += copy_from_mem(ptr, family_id);
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t k;
+ ptr += copy_from_mem(ptr, k);
+ uint16_t unused;
+ ptr += copy_from_mem(ptr, unused);
+ uint32_t dim;
+ ptr += copy_from_mem(ptr, dim);
+
+ //check_k(k);
Review Comment:
I am not sure we need to check, but I saw this commented-out line, which brings a question whether to remove it or implement.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] jmalkin commented on a diff in pull request #362: Density serialization
Posted by "jmalkin (via GitHub)" <gi...@apache.org>.
jmalkin commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170485428
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
+ ptr += copy_to_mem(pt.data(), ptr, pt_size);
+ }
+ }
+
+ if (ptr != end_ptr)
+ throw std::runtime_error("Actual output size does not equal expected output size");
+
+ return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+ const auto preamble_ints = read<uint8_t>(is);
+ const auto serial_version = read<uint8_t>(is);
+ const auto family_id = read<uint8_t>(is);
+ const auto flags_byte = read<uint8_t>(is);
+ const auto k = read<uint16_t>(is);
+ read<uint16_t>(is); // unused
+ const auto dim = read<uint32_t>(is);
+
+ //check_k(k); // do we have constraints?
+ check_serial_version(serial_version); // a little redundant with the header check
+ check_family_id(family_id);
+ check_header_validity(preamble_ints, flags_byte, serial_version);
+
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+ if (is_empty) {
+ return density_sketch(k, dim, kernel, allocator);
+ }
+
+ const auto num_retained = read<uint32_t>(is);
+ const auto n = read<uint64_t>(is);
+
+ // levels arrays
+ size_t pt_size = sizeof(T) * dim;
+ Levels levels(allocator);
+ int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+ while (num_to_read > 0) {
+ const auto level_size = read<uint32_t>(is);
+ Level lvl(allocator);
+ lvl.reserve(level_size);
+ for (uint32_t i = 0; i < level_size; ++i) {
+ Vector pt(dim, 0, allocator);
+ read(is, pt.data(), pt_size);
+ lvl.push_back(pt);
+ }
+ levels.push_back(lvl);
+ num_to_read -= lvl.size();
+ }
+
+ if (num_to_read != 0)
+ throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+ return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+ ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* end_ptr = static_cast<const char*>(bytes) + size;
+ uint8_t preamble_ints;
+ ptr += copy_from_mem(ptr, preamble_ints);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t family_id;
+ ptr += copy_from_mem(ptr, family_id);
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t k;
+ ptr += copy_from_mem(ptr, k);
+ uint16_t unused;
+ ptr += copy_from_mem(ptr, unused);
+ uint32_t dim;
+ ptr += copy_from_mem(ptr, dim);
+
+ //check_k(k);
Review Comment:
We don't check anything in the constructor so I dunno what the value range would be? It's uint16_t so it obviously can't be negative. We can check that it's strictly positive? It seems like there should be some minimum practical value given the need for compression, though.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization
Posted by "AlexanderSaydakov (via GitHub)" <gi...@apache.org>.
AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170729971
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +162,259 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
Review Comment:
const ref
##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +162,259 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
level.clear();
}
+/* Serialized sketch layout:
+ * Int || Start Byte Addr:
+ * Addr:
+ * || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+ * 0 || Preamble_Ints | SerVer | FamID | Flags |------- K -------|---- unused -----|
+ *
+ * || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
+ * 2 ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ * || 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
+ * 4 ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+ write(os, preamble_ints);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ write(os, ser_ver);
+ const uint8_t family = FAMILY_ID;
+ write(os, family);
+
+ // only empty is a valid flag
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ write(os, flags_byte);
+ write(os, k_);
+ const uint16_t unused = 0;
+ write(os, unused);
+ write(os, dim_);
+
+ if (is_empty())
+ return;
+
+ write(os, num_retained_);
+ write(os, n_);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+ write(os, level_size);
+ for (Vector pt : lvl) {
+ write(os, pt.data(), pt_size);
+ }
+ }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+
+ // pre-compute size
+ size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+ if (!is_empty())
+ for (Level lvl : levels_)
+ size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+ vector_bytes bytes(size, 0, levels_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(preamble_ints, ptr);
+ const uint8_t ser_ver = SERIAL_VERSION;
+ ptr += copy_to_mem(ser_ver, ptr);
+ const uint8_t family = FAMILY_ID;
+ ptr += copy_to_mem(family, ptr);
+
+ // empty is the only valid flat
+ const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem(k_, ptr);
+ ptr += sizeof(uint16_t); // 2 unused bytes
+ ptr += copy_to_mem(dim_, ptr);
+
+ if (is_empty())
+ return bytes;
+
+ ptr += copy_to_mem(num_retained_, ptr);
+ ptr += copy_to_mem(n_, ptr);
+
+ // levels array -- uint32_t since a single level may be larger than k
+ size_t pt_size = sizeof(T) * dim_;
+ for (Level lvl : levels_) {
+ ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+ for (Vector pt : lvl) {
Review Comment:
const fer
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org