You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2020/08/07 03:22:27 UTC

[incubator-datasketches-cpp] branch tuple_sketch updated: serialization to raw bytes with tests

This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git


The following commit(s) were added to refs/heads/tuple_sketch by this push:
     new 3acc466  serialization to raw bytes with tests
3acc466 is described below

commit 3acc46646d882fb810049492a2496991553a50fe
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Thu Aug 6 20:22:17 2020 -0700

    serialization to raw bytes with tests
---
 tuple/include/array_of_doubles_sketch.hpp      |   9 ++-
 tuple/include/array_of_doubles_sketch_impl.hpp | 107 ++++++++++++++++++++++---
 tuple/test/array_of_doubles_sketch_test.cpp    |  61 +++++++++++++-
 3 files changed, 164 insertions(+), 13 deletions(-)

diff --git a/tuple/include/array_of_doubles_sketch.hpp b/tuple/include/array_of_doubles_sketch.hpp
index cb1977e..ff87c14 100644
--- a/tuple/include/array_of_doubles_sketch.hpp
+++ b/tuple/include/array_of_doubles_sketch.hpp
@@ -80,11 +80,11 @@ public:
 template<typename A = std::allocator<std::vector<double>>>
 class compact_array_of_doubles_sketch: public compact_tuple_sketch<std::vector<double>, A> {
 public:
-  using Summary = std::vector<double>;
-  using Base = compact_tuple_sketch<Summary, A>;
+  using Base = compact_tuple_sketch<std::vector<double>, A>;
   using Entry = typename Base::Entry;
   using AllocEntry = typename Base::AllocEntry;
   using AllocU64 = typename Base::AllocU64;
+  using vector_bytes = typename Base::vector_bytes;
 
   static const uint8_t SERIAL_VERSION = 1;
   static const uint8_t SKETCH_FAMILY = 9;
@@ -97,13 +97,16 @@ public:
   uint8_t get_num_values() const;
 
   void serialize(std::ostream& os) const;
+  vector_bytes serialize(unsigned header_size_bytes = 0) const;
 
   static compact_array_of_doubles_sketch deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
+  static compact_array_of_doubles_sketch deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
+      const A& allocator = A());
 
   // for internal use
   compact_array_of_doubles_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values);
 private:
-  uint8_t num_values;
+  uint8_t num_values_;
 };
 
 } /* namespace datasketches */
diff --git a/tuple/include/array_of_doubles_sketch_impl.hpp b/tuple/include/array_of_doubles_sketch_impl.hpp
index 77e3b2b..b6371f2 100644
--- a/tuple/include/array_of_doubles_sketch_impl.hpp
+++ b/tuple/include/array_of_doubles_sketch_impl.hpp
@@ -51,16 +51,16 @@ update_array_of_doubles_sketch<A> update_array_of_doubles_sketch<A>::builder::bu
 template<typename A>
 template<typename S>
 compact_array_of_doubles_sketch<A>::compact_array_of_doubles_sketch(const S& other, bool ordered):
-Base(other, ordered), num_values(other.get_num_values()) {}
+Base(other, ordered), num_values_(other.get_num_values()) {}
 
 template<typename A>
 compact_array_of_doubles_sketch<A>::compact_array_of_doubles_sketch(bool is_empty, bool is_ordered,
     uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values):
-Base(is_empty, is_ordered, seed_hash, theta, std::move(entries)), num_values(num_values) {}
+Base(is_empty, is_ordered, seed_hash, theta, std::move(entries)), num_values_(num_values) {}
 
 template<typename A>
 uint8_t compact_array_of_doubles_sketch<A>::get_num_values() const {
-  return num_values;
+  return num_values_;
 }
 
 template<typename A>
@@ -79,7 +79,7 @@ void compact_array_of_doubles_sketch<A>::serialize(std::ostream& os) const {
     (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
   );
   os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
-  os.write(reinterpret_cast<const char*>(&num_values), sizeof(num_values));
+  os.write(reinterpret_cast<const char*>(&num_values_), sizeof(num_values_));
   const uint16_t seed_hash = this->get_seed_hash();
   os.write(reinterpret_cast<const char*>(&seed_hash), sizeof(seed_hash));
   os.write(reinterpret_cast<const char*>(&(this->theta_)), sizeof(uint64_t));
@@ -98,6 +98,47 @@ void compact_array_of_doubles_sketch<A>::serialize(std::ostream& os) const {
 }
 
 template<typename A>
+auto compact_array_of_doubles_sketch<A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+  const uint8_t preamble_longs = 1;
+  const size_t size = header_size_bytes + 16 // preamble and theta
+      + (this->entries_.size() > 0 ? 8 : 0)
+      + (sizeof(uint64_t) + sizeof(double) * num_values_) * this->entries_.size();
+  vector_bytes bytes(size, 0, this->entries_.get_allocator());
+  uint8_t* ptr = bytes.data() + header_size_bytes;
+
+  ptr += copy_to_mem(&preamble_longs, ptr, sizeof(preamble_longs));
+  const uint8_t serial_version = SERIAL_VERSION;
+  ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+  const uint8_t family = SKETCH_FAMILY;
+  ptr += copy_to_mem(&family, ptr, sizeof(family));
+  const uint8_t type = SKETCH_TYPE;
+  ptr += copy_to_mem(&type, ptr, sizeof(type));
+  const uint8_t flags_byte(
+    (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+    (this->get_num_retained() ? 1 << flags::HAS_ENTRIES : 0) |
+    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+  );
+  ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
+  ptr += copy_to_mem(&num_values_, ptr, sizeof(num_values_));
+  const uint16_t seed_hash = this->get_seed_hash();
+  ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash));
+  ptr += copy_to_mem(&(this->theta_), ptr, sizeof(uint64_t));
+  if (this->get_num_retained() > 0) {
+    const uint32_t num_entries = this->entries_.size();
+    ptr += copy_to_mem(&num_entries, ptr, sizeof(num_entries));
+    const uint32_t unused32 = 0;
+    ptr += copy_to_mem(&unused32, ptr, sizeof(unused32));
+    for (const auto& it: this->entries_) {
+      ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
+    }
+    for (const auto& it: this->entries_) {
+      ptr += copy_to_mem(it.second.data(), ptr, it.second.size() * sizeof(double));
+    }
+  }
+  return bytes;
+}
+
+template<typename A>
 compact_array_of_doubles_sketch<A> compact_array_of_doubles_sketch<A>::deserialize(std::istream& is, uint64_t seed, const A& allocator) {
   uint8_t preamble_longs;
   is.read(reinterpret_cast<char*>(&preamble_longs), sizeof(preamble_longs));
@@ -113,17 +154,17 @@ compact_array_of_doubles_sketch<A> compact_array_of_doubles_sketch<A>::deseriali
   is.read(reinterpret_cast<char*>(&num_values), sizeof(num_values));
   uint16_t seed_hash;
   is.read(reinterpret_cast<char*>(&seed_hash), sizeof(seed_hash));
+  checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
   checker<true>::check_sketch_family(family, SKETCH_FAMILY);
   checker<true>::check_sketch_type(type, SKETCH_TYPE);
-  checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
   const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
   if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
 
   uint64_t theta;
   is.read(reinterpret_cast<char*>(&theta), sizeof(theta));
   std::vector<Entry, AllocEntry> entries(allocator);
-  uint32_t num_entries = 0;
   if (has_entries) {
+    uint32_t num_entries;
     is.read(reinterpret_cast<char*>(&num_entries), sizeof(num_entries));
     uint32_t unused32;
     is.read(reinterpret_cast<char*>(&unused32), sizeof(unused32));
@@ -131,9 +172,9 @@ compact_array_of_doubles_sketch<A> compact_array_of_doubles_sketch<A>::deseriali
     std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
     is.read(reinterpret_cast<char*>(keys.data()), num_entries * sizeof(uint64_t));
     for (size_t i = 0; i < num_entries; ++i) {
-      Summary s(num_values, 0, allocator);
-      is.read(reinterpret_cast<char*>(s.data()), num_values * sizeof(double));
-      entries.push_back(Entry(keys[i], std::move(s)));
+      std::vector<double> summary(num_values, 0, allocator);
+      is.read(reinterpret_cast<char*>(summary.data()), num_values * sizeof(double));
+      entries.push_back(Entry(keys[i], std::move(summary)));
     }
   }
   if (!is.good()) throw std::runtime_error("error reading from std::istream");
@@ -142,4 +183,52 @@ compact_array_of_doubles_sketch<A> compact_array_of_doubles_sketch<A>::deseriali
   return compact_array_of_doubles_sketch(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
 }
 
+template<typename A>
+compact_array_of_doubles_sketch<A> compact_array_of_doubles_sketch<A>::deserialize(const void* bytes, size_t size, uint64_t seed, const A& allocator) {
+  ensure_minimum_memory(size, 16);
+  const char* ptr = static_cast<const char*>(bytes);
+  uint8_t preamble_longs;
+  ptr += copy_from_mem(ptr, &preamble_longs, sizeof(preamble_longs));
+  uint8_t serial_version;
+  ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+  uint8_t family;
+  ptr += copy_from_mem(ptr, &family, sizeof(family));
+  uint8_t type;
+  ptr += copy_from_mem(ptr, &type, sizeof(type));
+  uint8_t flags_byte;
+  ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+  uint8_t num_values;
+  ptr += copy_from_mem(ptr, &num_values, sizeof(num_values));
+  uint16_t seed_hash;
+  ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash));
+  checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+  checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+  checker<true>::check_sketch_type(type, SKETCH_TYPE);
+  const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
+  if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+  uint64_t theta;
+  ptr += copy_from_mem(ptr, &theta, sizeof(theta));
+  std::vector<Entry, AllocEntry> entries(allocator);
+  if (has_entries) {
+    ensure_minimum_memory(size, 24);
+    uint32_t num_entries;
+    ptr += copy_from_mem(ptr, &num_entries, sizeof(num_entries));
+    uint32_t unused32;
+    ptr += copy_from_mem(ptr, &unused32, sizeof(unused32));
+    ensure_minimum_memory(size, 24 + (sizeof(uint64_t) + sizeof(double) * num_values) * num_entries);
+    entries.reserve(num_entries);
+    std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
+    ptr += copy_from_mem(ptr, keys.data(), sizeof(uint64_t) * num_entries);
+    for (size_t i = 0; i < num_entries; ++i) {
+      std::vector<double> summary(num_values, 0, allocator);
+      ptr += copy_from_mem(ptr, summary.data(), num_values * sizeof(double));
+      entries.push_back(Entry(keys[i], std::move(summary)));
+    }
+  }
+  const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+  const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+  return compact_array_of_doubles_sketch(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
+}
+
 } /* namespace datasketches */
diff --git a/tuple/test/array_of_doubles_sketch_test.cpp b/tuple/test/array_of_doubles_sketch_test.cpp
index 539962a..711ead1 100644
--- a/tuple/test/array_of_doubles_sketch_test.cpp
+++ b/tuple/test/array_of_doubles_sketch_test.cpp
@@ -157,7 +157,7 @@ TEST_CASE("aod sketch: serialization compatibility with java - exact mode with t
   }
 }
 
-TEST_CASE("aod sketch: serialize deserialize - estimation mode", "[tuple_sketch]") {
+TEST_CASE("aod sketch: stream serialize deserialize - estimation mode", "[tuple_sketch]") {
   auto update_sketch = update_array_of_doubles_sketch<>::builder(2).build();
   std::vector<double> a = {1, 2};
   for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
@@ -187,4 +187,63 @@ TEST_CASE("aod sketch: serialize deserialize - estimation mode", "[tuple_sketch]
   }
 }
 
+TEST_CASE("aod sketch: bytes serialize deserialize - estimation mode", "[tuple_sketch]") {
+  auto update_sketch = update_array_of_doubles_sketch<>::builder(2).build();
+  std::vector<double> a = {1, 2};
+  for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+  auto compact_sketch = update_sketch.compact();
+
+  auto bytes = compact_sketch.serialize();
+  std::stringstream ss;
+  ss.exceptions(std::ios::failbit | std::ios::badbit);
+  ss.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+  auto deserialized_sketch = compact_array_of_doubles_sketch<>::deserialize(ss);
+  REQUIRE(compact_sketch.get_num_retained() == deserialized_sketch.get_num_retained());
+  REQUIRE(compact_sketch.get_theta() == Approx(deserialized_sketch.get_theta()).margin(1e-10));
+  REQUIRE(compact_sketch.get_estimate() == Approx(deserialized_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(2) == Approx(deserialized_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(2) == Approx(deserialized_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(3) == Approx(deserialized_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(3) == Approx(deserialized_sketch.get_upper_bound(3)).margin(1e-10));
+  // sketches must be ordered and the iteration sequence must match exactly
+  auto it = deserialized_sketch.begin();
+  for (const auto& entry: compact_sketch) {
+    REQUIRE(entry.first == (*it).first);
+    REQUIRE(entry.second.size() == 2);
+    REQUIRE(entry.second[0] == (*it).second[0]);
+    REQUIRE(entry.second[1] == (*it).second[1]);
+    ++it;
+  }
+}
+
+TEST_CASE("aod sketch: bytes to stream serialize deserialize - estimation mode", "[tuple_sketch]") {
+  auto update_sketch = update_array_of_doubles_sketch<>::builder(2).build();
+  std::vector<double> a = {1, 2};
+  for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+  auto compact_sketch = update_sketch.compact();
+
+  auto bytes = compact_sketch.serialize();
+  auto deserialized_sketch = compact_array_of_doubles_sketch<>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(compact_sketch.get_num_retained() == deserialized_sketch.get_num_retained());
+  REQUIRE(compact_sketch.get_theta() == Approx(deserialized_sketch.get_theta()).margin(1e-10));
+  REQUIRE(compact_sketch.get_estimate() == Approx(deserialized_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(2) == Approx(deserialized_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(2) == Approx(deserialized_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(compact_sketch.get_lower_bound(3) == Approx(deserialized_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(compact_sketch.get_upper_bound(3) == Approx(deserialized_sketch.get_upper_bound(3)).margin(1e-10));
+  // sketches must be ordered and the iteration sequence must match exactly
+  auto it = deserialized_sketch.begin();
+  for (const auto& entry: compact_sketch) {
+    REQUIRE(entry.first == (*it).first);
+    REQUIRE(entry.second.size() == 2);
+    REQUIRE(entry.second[0] == (*it).second[0]);
+    REQUIRE(entry.second[1] == (*it).second[1]);
+    ++it;
+  }
+}
+
 } /* namespace datasketches */


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org