[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a diff in pull request #362: Density serialization

["AlexanderSaydakov (via GitHub)" <gi...@apache.org>]

AlexanderSaydakov commented on code in PR #362:
URL: https://github.com/apache/datasketches-cpp/pull/362#discussion_r1170478518


##########
density/include/density_sketch_impl.hpp:
##########
@@ -146,6 +158,253 @@ void density_sketch<T, K, A>::compact_level(unsigned height) {
   level.clear();
 }
 
+/* Serialized sketch layout:
+ * Int  || Start Byte Addr:
+ * Addr:
+ *      ||       0        |    1   |    2   |    3   |    4   |    5   |    6   |    7   |
+ *  0   || Preamble_Ints  | SerVer | FamID  |  Flags |------- K -------|---- unused -----|
+ *
+ *      ||       8        |    9   |   10   |   11   |   12   |   13   |   14   |   15   |
+ *  2   ||------------- Num Dimensions --------------|------ Num Retained Items ---------|
+ *
+ *      ||       16       |   17   |   18   |   19   |   20   |   21   |   22   |   23   |
+ *  4   ||---------------------------Items Seen Count (N)--------------------------------|
+ *
+ * Ints 2 and 3 are omitted when the sketch is empty, meaning Num Dimensions is stored at
+ * offset 8 in that case. Otherwise, Int 5 is the start of level data, consisting of the
+ * size of the level (as a uint32 value) followed by that number of points, with 
+ * Num Dimensions per point.
+ */
+
+template<typename T, typename K, typename A>
+void density_sketch<T, K, A>::serialize(std::ostream& os) const {
+  const uint8_t preamble_ints = is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG;
+  write(os, preamble_ints);
+  const uint8_t ser_ver = SERIAL_VERSION;
+  write(os, ser_ver);
+  const uint8_t family = FAMILY_ID;
+  write(os, family);
+
+  // only empty is a valid flag
+  const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+  write(os, flags_byte);
+  write(os, k_);
+  const uint16_t unused = 0;
+  write(os, unused);
+  write(os, dim_);
+
+  if (is_empty())
+    return;
+
+  write(os, num_retained_);
+  write(os, n_);
+
+  // levels array -- uint32_t since a single level may be larger than k
+  size_t pt_size = sizeof(T) * dim_;
+  for (Level lvl : levels_) {
+    const uint32_t level_size = static_cast<uint32_t>(lvl.size());
+    write(os, level_size);
+    for (Vector pt : lvl) {
+      write(os, pt.data(), pt_size);
+    }
+  }
+}
+
+template<typename T, typename K, typename A>
+auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+  const uint8_t preamble_ints = (is_empty() ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_LONG);
+  
+  // pre-compute size
+  size_t size = header_size_bytes + preamble_ints * sizeof(uint32_t);
+  if (!is_empty())
+    for (Level lvl : levels_)
+      size += sizeof(uint32_t) + (lvl.size() * dim_ * sizeof(T));
+
+  vector_bytes bytes(size, 0, levels_.get_allocator());
+  uint8_t* ptr = bytes.data() + header_size_bytes;
+  const uint8_t* end_ptr = ptr + size;
+  
+  ptr += copy_to_mem(preamble_ints, ptr);
+  const uint8_t ser_ver = SERIAL_VERSION;
+  ptr += copy_to_mem(ser_ver, ptr);
+  const uint8_t family = FAMILY_ID;
+  ptr += copy_to_mem(family, ptr);
+
+  // empty is the only valid flat
+  const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
+  ptr += copy_to_mem(flags_byte, ptr);
+  ptr += copy_to_mem(k_, ptr);
+  ptr += sizeof(uint16_t); // 2 unused bytes
+  ptr += copy_to_mem(dim_, ptr);
+  
+  if (is_empty())
+    return bytes;
+
+  ptr += copy_to_mem(num_retained_, ptr);
+  ptr += copy_to_mem(n_, ptr);
+
+  // levels array -- uint32_t since a single level may be larger than k
+  size_t pt_size = sizeof(T) * dim_;
+  for (Level lvl : levels_) {
+    ptr += copy_to_mem(static_cast<uint32_t>(lvl.size()), ptr);
+    for (Vector pt : lvl) {
+      ptr += copy_to_mem(pt.data(), ptr, pt_size);
+    }
+  }
+
+  if (ptr != end_ptr)
+    throw std::runtime_error("Actual output size does not equal expected output size");
+
+  return bytes;
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(std::istream& is, const K& kernel, const A& allocator) {
+  const auto preamble_ints = read<uint8_t>(is);
+  const auto serial_version = read<uint8_t>(is);
+  const auto family_id = read<uint8_t>(is);
+  const auto flags_byte = read<uint8_t>(is);
+  const auto k = read<uint16_t>(is);
+  read<uint16_t>(is); // unused
+  const auto dim = read<uint32_t>(is);
+
+  //check_k(k); // do we have constraints?
+  check_serial_version(serial_version); // a little redundant with the header check
+  check_family_id(family_id);
+  check_header_validity(preamble_ints, flags_byte, serial_version);
+
+  if (!is.good()) throw std::runtime_error("error reading from std::istream");
+  const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
+  if (is_empty) {
+    return density_sketch(k, dim, kernel, allocator);
+  }
+
+  const auto num_retained = read<uint32_t>(is);
+  const auto n = read<uint64_t>(is);
+
+  // levels arrays
+  size_t pt_size = sizeof(T) * dim;
+  Levels levels(allocator);
+  int64_t num_to_read = num_retained; // num_retrained is uint32_t so this allows error checking
+  while (num_to_read > 0) {
+    const auto level_size = read<uint32_t>(is);
+    Level lvl(allocator);
+    lvl.reserve(level_size);
+    for (uint32_t i = 0; i < level_size; ++i) {
+      Vector pt(dim, 0, allocator);
+      read(is, pt.data(), pt_size);
+      lvl.push_back(pt);
+    }
+    levels.push_back(lvl);
+    num_to_read -= lvl.size();
+  }
+
+  if (num_to_read != 0)
+    throw std::runtime_error("Error deserializing sketch: Incorrect number of items read");
+  if (!is.good()) throw std::runtime_error("error reading from std::istream");
+
+  return density_sketch(k, dim, num_retained, n, std::move(levels), kernel);
+}
+
+template<typename T, typename K, typename A>
+density_sketch<T, K, A> density_sketch<T, K, A>::deserialize(const void* bytes, size_t size, const K& kernel, const A& allocator) {
+  ensure_minimum_memory(size, PREAMBLE_INTS_SHORT * sizeof(uint32_t));
+  const char* ptr = static_cast<const char*>(bytes);
+  const char* end_ptr = static_cast<const char*>(bytes) + size;
+  uint8_t preamble_ints;
+  ptr += copy_from_mem(ptr, preamble_ints);
+  uint8_t serial_version;
+  ptr += copy_from_mem(ptr, serial_version);
+  uint8_t family_id;
+  ptr += copy_from_mem(ptr, family_id);
+  uint8_t flags_byte;
+  ptr += copy_from_mem(ptr, flags_byte);
+  uint16_t k;
+  ptr += copy_from_mem(ptr, k);
+  uint16_t unused;
+  ptr += copy_from_mem(ptr, unused);
+  uint32_t dim;
+  ptr += copy_from_mem(ptr, dim);
+
+  //check_k(k);

Review Comment:
   do we need to check K here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org