You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2020/08/14 23:46:41 UTC

[incubator-datasketches-cpp] branch tuple_sketch updated (3acc466 -> 67d7203)

This is an automated email from the ASF dual-hosted git repository.

alsay pushed a change to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git.


    from 3acc466  serialization to raw bytes with tests
     new 99cc639  allocation fix
     new 418b974  stream type change and serialization order change
     new b3f5439  better allocation test
     new bae2fb0  disabled print
     new 6906132  avoid swapping uninitialized payloads in rebuild
     new 013b0d0  renamed method
     add fc2a1b7  Merge pull request #2 from apache/master
     add 30437a0  Fixed vector_of_kll_sketches check for number of elements to consider the last axis
     add 7e210a8  vector_of_kll_sketches: fixed 2D updates. Added unit tests for 2D and 3D updates.
     add 199829b  Merge pull request #161 from mdhimes/master
     add 592506e  better way to specify ranges
     add c6fe453  Merge pull request #162 from apache/msvc_compatibility
     add 0ad6212  simplify and avoid iterating past the end
     add 762904b  Merge pull request #163 from apache/msvc_compatibility
     add e5fea0e  add array input to kll sketches in python
     add c66bde3  Merge pull request #164 from apache/py_kll_array_input
     add 337fbd7  Issue-165: Remove c-style const casts from kll/ and common directories
     add c67d92f  Merge pull request #166 from gaborkaszab/const_cast
     add 2946e31  Add empty build/ directory to repo, ensure it stays empty
     add 55722ca  Update workflows to skip creating build/, add python descriptions
     add 2ab2380  Merge pull request #167 from apache/python_description
     add 7cafbef  rewritten quantile calculator
     add 3135dc4  avoid warnings, unified style
     add 608a741  removed unused include
     add fa95167  Merge pull request #168 from apache/kll_merge_blocks_fix
     new 67d7203  Merge branch 'master' into tuple_sketch

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/build_cmake.yml               |   2 +-
 .github/workflows/code_coverage.yml             |   2 +-
 .gitignore                                      |   1 -
 build/.gitignore                                |   7 ++
 common/include/serde.hpp                        |   9 +-
 common/test/test_type.hpp                       |  44 ++++---
 kll/include/kll_quantile_calculator.hpp         |  37 +++---
 kll/include/kll_quantile_calculator_impl.hpp    | 159 ++++++++++--------------
 kll/include/kll_sketch_impl.hpp                 |  33 ++---
 python/src/cpc_wrapper.cpp                      |  41 +++---
 python/src/fi_wrapper.cpp                       |  48 ++++---
 python/src/hll_wrapper.cpp                      | 101 +++++++++------
 python/src/kll_wrapper.cpp                      | 115 ++++++++++++++---
 python/src/theta_wrapper.cpp                    |  75 +++++++----
 python/src/vector_of_kll.cpp                    |  13 +-
 python/src/vo_wrapper.cpp                       |  44 ++++---
 python/tests/kll_test.py                        |   8 +-
 python/tests/vector_of_kll_test.py              |  48 +++++++
 theta/include/theta_intersection_impl.hpp       |   4 +-
 theta/include/theta_sketch_impl.hpp             |   2 +-
 theta/include/theta_union_impl.hpp              |   2 +-
 tuple/include/theta_update_sketch_base.hpp      |   1 +
 tuple/include/theta_update_sketch_base_impl.hpp |  44 +++++--
 tuple/include/tuple_sketch.hpp                  |   6 +-
 tuple/include/tuple_sketch_impl.hpp             |  18 ++-
 tuple/test/tuple_sketch_allocation_test.cpp     |  91 ++++++++------
 tuple/test/tuple_sketch_test.cpp                |  42 +------
 tuple/test/tuple_union_test.cpp                 |   6 +-
 28 files changed, 617 insertions(+), 386 deletions(-)
 create mode 100644 build/.gitignore


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 01/07: allocation fix

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit 99cc63939914b6457de6f71b84c4957f09ef16c0
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:29:33 2020 -0700

    allocation fix
---
 tuple/include/theta_update_sketch_base_impl.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index cf6b46d..1cf88fb 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -59,7 +59,7 @@ entries_(nullptr)
     entries_ = allocator_.allocate(size);
     for (size_t i = 0; i < size; ++i) {
       if (EK()(other.entries_[i]) != 0) {
-        entries_[i] = other.entries_[i];
+        new (&entries_[i]) EN(other.entries_[i]);
       } else {
         EK()(entries_[i]) = 0;
       }
@@ -212,7 +212,7 @@ template<typename EN, typename EK, typename A>
 void theta_update_sketch_base<EN, EK, A>::rebuild() {
   const size_t size = 1 << lg_cur_size_;
   const uint32_t pivot = (1 << lg_nom_size_) + size - num_entries_;
-  std::nth_element(&entries_[0], &entries_[pivot], &entries_[size], comparator());
+  std::nth_element(entries_, entries_ + pivot, entries_ + size, comparator());
   this->theta_ = EK()(entries_[pivot]);
   EN* old_entries = entries_;
   entries_ = allocator_.allocate(size);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 02/07: stream type change and serialization order change

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit 418b97415f19e1f15b25b6cfe36bac9d0596f3d5
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:31:35 2020 -0700

    stream type change and serialization order change
---
 tuple/include/tuple_sketch.hpp      |  6 +++---
 tuple/include/tuple_sketch_impl.hpp | 18 ++++++++----------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/tuple/include/tuple_sketch.hpp b/tuple/include/tuple_sketch.hpp
index 7e2475a..45f0de5 100644
--- a/tuple/include/tuple_sketch.hpp
+++ b/tuple/include/tuple_sketch.hpp
@@ -142,7 +142,7 @@ public:
   virtual const_iterator end() const = 0;
 
 protected:
-  virtual void print_specifics(std::ostringstream& os) const = 0;
+  virtual void print_specifics(std::basic_ostream<char>& os) const = 0;
 
   static uint16_t get_seed_hash(uint64_t seed);
 
@@ -332,7 +332,7 @@ protected:
   // for builder
   update_tuple_sketch(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
 
-  virtual void print_specifics(std::ostringstream& os) const;
+  virtual void print_specifics(std::basic_ostream<char>& os) const;
 };
 
 // compact sketch
@@ -455,7 +455,7 @@ protected:
     bool destroy;
   };
 
-  virtual void print_specifics(std::ostringstream& os) const;
+  virtual void print_specifics(std::basic_ostream<char>& os) const;
 
 };
 
diff --git a/tuple/include/tuple_sketch_impl.hpp b/tuple/include/tuple_sketch_impl.hpp
index f0c5b50..fff3b2c 100644
--- a/tuple/include/tuple_sketch_impl.hpp
+++ b/tuple/include/tuple_sketch_impl.hpp
@@ -238,7 +238,7 @@ compact_tuple_sketch<S, A> update_tuple_sketch<S, U, P, A>::compact(bool ordered
 }
 
 template<typename S, typename U, typename P, typename A>
-void update_tuple_sketch<S, U, P, A>::print_specifics(std::ostringstream& os) const {
+void update_tuple_sketch<S, U, P, A>::print_specifics(std::basic_ostream<char>& os) const {
   os << "   lg nominal size      : " << (int) map_.lg_nom_size_ << std::endl;
   os << "   lg current size      : " << (int) map_.lg_cur_size_ << std::endl;
   os << "   resize factor        : " << (1 << map_.rf_) << std::endl;
@@ -397,8 +397,6 @@ auto compact_tuple_sketch<S, A>::serialize(unsigned header_size_bytes, const Ser
     }
     for (const auto& it: entries_) {
       ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
-    }
-    for (const auto& it: entries_) {
       ptr += sd.serialize(ptr, end_ptr - ptr, &it.second, 1);
     }
   }
@@ -502,14 +500,14 @@ compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(const void* b
   std::vector<Entry, AllocEntry> entries(allocator);
   if (!is_empty) {
     entries.reserve(num_entries);
-    std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
-    ptr += copy_from_mem(ptr, keys.data(), keys_size_bytes);
     A alloc(allocator);
-    std::unique_ptr<S, deleter_of_summaries> summaries(alloc.allocate(num_entries), deleter_of_summaries(num_entries, false));
-    ptr += sd.deserialize(ptr, base + size - ptr, summaries.get(), num_entries);
-    summaries.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
+    std::unique_ptr<S, deleter_of_summaries> summary(alloc.allocate(1), deleter_of_summaries(1, false));
     for (size_t i = 0; i < num_entries; ++i) {
-      entries.push_back(Entry(keys[i], std::move(summaries.get()[i])));
+      uint64_t key;
+      ptr += copy_from_mem(ptr, &key, sizeof(key));
+      ptr += sd.deserialize(ptr, base + size - ptr, summary.get(), 1);
+      entries.push_back(Entry(key, std::move(*summary)));
+      summary->~S();
     }
   }
   const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
@@ -537,7 +535,7 @@ auto compact_tuple_sketch<S, A>::end() const -> const_iterator {
 }
 
 template<typename S, typename A>
-void compact_tuple_sketch<S, A>::print_specifics(std::ostringstream&) const {}
+void compact_tuple_sketch<S, A>::print_specifics(std::basic_ostream<char>&) const {}
 
 // builder
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 04/07: disabled print

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit bae2fb029391acec38595d1f080ae3c37a43595e
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:33:24 2020 -0700

    disabled print
---
 tuple/test/tuple_union_test.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tuple/test/tuple_union_test.cpp b/tuple/test/tuple_union_test.cpp
index 38289fd..00127ea 100644
--- a/tuple/test/tuple_union_test.cpp
+++ b/tuple/test/tuple_union_test.cpp
@@ -29,7 +29,7 @@ TEST_CASE("tuple_union float: empty", "[tuple union]") {
   auto u = tuple_union<float>::builder().build();
   u.update(update_sketch);
   auto result = u.get_result();
-  std::cout << result.to_string(true);
+//  std::cout << result.to_string(true);
   REQUIRE(result.is_empty());
   REQUIRE(result.get_num_retained() == 0);
   REQUIRE(!result.is_estimation_mode());
@@ -38,14 +38,14 @@ TEST_CASE("tuple_union float: empty", "[tuple union]") {
 
 TEST_CASE("tuple_union float: non-empty no retained entries", "[tuple union]") {
   auto update_sketch = update_tuple_sketch<float>::builder().set_p(0.001).build();
-  std::cout << update_sketch.to_string();
+//  std::cout << update_sketch.to_string();
   update_sketch.update(1, 1);
   REQUIRE(!update_sketch.is_empty());
   REQUIRE(update_sketch.get_num_retained() == 0);
   auto u = tuple_union<float>::builder().build();
   u.update(update_sketch);
   auto result = u.get_result();
-  std::cout << result.to_string(true);
+//  std::cout << result.to_string(true);
   REQUIRE(!result.is_empty());
   REQUIRE(result.get_num_retained() == 0);
   REQUIRE(result.is_estimation_mode());


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 07/07: Merge branch 'master' into tuple_sketch

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit 67d7203595229704e7c0575dd9b1eef1f7e87b6e
Merge: 013b0d0 fa95167
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 16:36:14 2020 -0700

    Merge branch 'master' into tuple_sketch

 .github/workflows/build_cmake.yml            |   2 +-
 .github/workflows/code_coverage.yml          |   2 +-
 .gitignore                                   |   1 -
 build/.gitignore                             |   7 ++
 common/include/serde.hpp                     |   9 +-
 kll/include/kll_quantile_calculator.hpp      |  37 ++++---
 kll/include/kll_quantile_calculator_impl.hpp | 159 ++++++++++++---------------
 kll/include/kll_sketch_impl.hpp              |  33 +++---
 python/src/cpc_wrapper.cpp                   |  41 ++++---
 python/src/fi_wrapper.cpp                    |  48 +++++---
 python/src/hll_wrapper.cpp                   | 101 ++++++++++-------
 python/src/kll_wrapper.cpp                   | 115 +++++++++++++++----
 python/src/theta_wrapper.cpp                 |  75 ++++++++-----
 python/src/vector_of_kll.cpp                 |  13 ++-
 python/src/vo_wrapper.cpp                    |  44 +++++---
 python/tests/kll_test.py                     |   8 +-
 python/tests/vector_of_kll_test.py           |  48 ++++++++
 theta/include/theta_intersection_impl.hpp    |   4 +-
 theta/include/theta_sketch_impl.hpp          |   2 +-
 theta/include/theta_union_impl.hpp           |   2 +-
 20 files changed, 481 insertions(+), 270 deletions(-)

diff --cc common/include/serde.hpp
index b8231c9,d0819d8..73e0901
--- a/common/include/serde.hpp
+++ b/common/include/serde.hpp
@@@ -47,10 -47,10 +47,10 @@@ template<typename T, typename Enable = 
  // with LongsSketch and ItemsSketch<Long> with ArrayOfLongsSerDe in Java
  template<typename T>
  struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
 -  void serialize(std::ostream& os, const T* items, unsigned num) {
 +  void serialize(std::ostream& os, const T* items, unsigned num) const {
      bool failure = false;
      try {
-       os.write((char*)items, sizeof(T) * num);
+       os.write(reinterpret_cast<const char*>(items), sizeof(T) * num);
      } catch (std::ostream::failure& e) {
        failure = true;
      }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 06/07: renamed method

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit 013b0d01887c966e821d8b995978e48eb4c6e063
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 09:57:27 2020 -0700

    renamed method
---
 tuple/include/theta_update_sketch_base.hpp      | 2 +-
 tuple/include/theta_update_sketch_base_impl.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tuple/include/theta_update_sketch_base.hpp b/tuple/include/theta_update_sketch_base.hpp
index 5501581..e65a93d 100644
--- a/tuple/include/theta_update_sketch_base.hpp
+++ b/tuple/include/theta_update_sketch_base.hpp
@@ -87,7 +87,7 @@ struct theta_update_sketch_base {
 
   static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
   static inline uint32_t get_stride(uint64_t key, uint8_t lg_size);
-  static void consolidate_empty(Entry* entries, size_t size, size_t num);
+  static void consolidate_non_empty(Entry* entries, size_t size, size_t num);
 };
 
 // builder
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index 7047f98..baf94ea 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -238,7 +238,7 @@ void theta_update_sketch_base<EN, EK, A>::trim() {
 }
 
 template<typename EN, typename EK, typename A>
-void theta_update_sketch_base<EN, EK, A>::consolidate_empty(EN* entries, size_t size, size_t num) {
+void theta_update_sketch_base<EN, EK, A>::consolidate_non_empty(EN* entries, size_t size, size_t num) {
   // find the first empty slot
   size_t i = 0;
   while (i < size) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 05/07: avoid swapping uninitialized payloads in rebuild

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit 690613213a4e8acdd62ab3355bdb716da47a28bd
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 09:54:47 2020 -0700

    avoid swapping uninitialized payloads in rebuild
---
 tuple/include/theta_update_sketch_base.hpp      |  1 +
 tuple/include/theta_update_sketch_base_impl.hpp | 42 +++++++++++++++++++------
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/tuple/include/theta_update_sketch_base.hpp b/tuple/include/theta_update_sketch_base.hpp
index 15b6acd..5501581 100644
--- a/tuple/include/theta_update_sketch_base.hpp
+++ b/tuple/include/theta_update_sketch_base.hpp
@@ -87,6 +87,7 @@ struct theta_update_sketch_base {
 
   static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
   static inline uint32_t get_stride(uint64_t key, uint8_t lg_size);
+  static void consolidate_empty(Entry* entries, size_t size, size_t num);
 };
 
 // builder
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index 1cf88fb..7047f98 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -208,22 +208,26 @@ void theta_update_sketch_base<EN, EK, A>::resize() {
   allocator_.deallocate(old_entries, old_size);
 }
 
+// assumes number of entries > nominal size
 template<typename EN, typename EK, typename A>
 void theta_update_sketch_base<EN, EK, A>::rebuild() {
   const size_t size = 1 << lg_cur_size_;
-  const uint32_t pivot = (1 << lg_nom_size_) + size - num_entries_;
-  std::nth_element(entries_, entries_ + pivot, entries_ + size, comparator());
-  this->theta_ = EK()(entries_[pivot]);
+  const uint32_t nominal_size = 1 << lg_nom_size_;
+
+  // empty entries have uninitialized payloads
+  // TODO: avoid this for empty or trivial payloads (arithmetic types)
+  consolidate_non_empty(entries_, size, num_entries_);
+
+  std::nth_element(entries_, entries_ + nominal_size, entries_ + num_entries_, comparator());
+  this->theta_ = EK()(entries_[nominal_size]);
   EN* old_entries = entries_;
   entries_ = allocator_.allocate(size);
   for (size_t i = 0; i < size; ++i) EK()(entries_[i]) = 0;
   num_entries_ = 0;
-  for (size_t i = 0; i < size; ++i) {
-    const uint64_t key = EK()(old_entries[i]);
-    if (key != 0 && key < this->theta_) {
-      insert(find(key).first, std::move(old_entries[i])); // consider a special insert with no comparison
-      old_entries[i].~EN();
-    }
+  // relies on consolidating non-empty entries to the front
+  for (size_t i = 0; i < nominal_size; ++i) {
+    insert(find(EK()(old_entries[i])).first, std::move(old_entries[i])); // consider a special insert with no comparison
+    old_entries[i].~EN();
   }
   allocator_.deallocate(old_entries, size);
 }
@@ -233,6 +237,26 @@ void theta_update_sketch_base<EN, EK, A>::trim() {
   if (num_entries_ > static_cast<uint32_t>(1 << lg_nom_size_)) rebuild();
 }
 
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::consolidate_empty(EN* entries, size_t size, size_t num) {
+  // find the first empty slot
+  size_t i = 0;
+  while (i < size) {
+    if (EK()(entries[i]) == 0) break;
+    ++i;
+  }
+  // scan the rest and move non-empty entries to the front
+  for (size_t j = i + 1; j < size; ++j) {
+    if (EK()(entries[j]) != 0) {
+      new (&entries[i]) EN(std::move(entries[j]));
+      entries[j].~EN();
+      EK()(entries[j]) = 0;
+      ++i;
+      if (i == num) break;
+    }
+  }
+}
+
 // builder
 
 template<typename Derived>


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-cpp] 03/07: better allocation test

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git

commit b3f5439ee85e6d0e210c3fb768209dbfc26245f0
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:33:04 2020 -0700

    better allocation test
---
 common/test/test_type.hpp                   | 44 +++++++++-----
 tuple/test/tuple_sketch_allocation_test.cpp | 91 +++++++++++++++++------------
 tuple/test/tuple_sketch_test.cpp            | 42 +------------
 3 files changed, 87 insertions(+), 90 deletions(-)

diff --git a/common/test/test_type.hpp b/common/test/test_type.hpp
index cbd1a2e..18be598 100644
--- a/common/test/test_type.hpp
+++ b/common/test/test_type.hpp
@@ -24,38 +24,51 @@
 
 namespace datasketches {
 
-class test_type {
-  static const bool DEBUG = true;
+template<typename A>
+class test_type_alloc {
+  static const bool DEBUG = false;
 public:
   // no default constructor should be required
-  test_type(int value): value(value) {
+  test_type_alloc(int value): value_ptr(A().allocate(1)) {
     if (DEBUG) std::cerr << "test_type constructor" << std::endl;
+    *value_ptr = value;
   }
-  ~test_type() {
+  ~test_type_alloc() {
     if (DEBUG) std::cerr << "test_type destructor" << std::endl;
+    if (value_ptr != nullptr) A().deallocate(value_ptr, 1);
   }
-  test_type(const test_type& other): value(other.value) {
+  test_type_alloc(const test_type_alloc& other): value_ptr(A().allocate(1)) {
     if (DEBUG) std::cerr << "test_type copy constructor" << std::endl;
+    *value_ptr = *other.value_ptr;
   }
   // noexcept is important here so that, for instance, std::vector could move this type
-  test_type(test_type&& other) noexcept : value(other.value) {
+  test_type_alloc(test_type_alloc&& other) noexcept : value_ptr(nullptr) {
     if (DEBUG) std::cerr << "test_type move constructor" << std::endl;
+    if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+    std::swap(value_ptr, other.value_ptr);
   }
-  test_type& operator=(const test_type& other) {
+  test_type_alloc& operator=(const test_type_alloc& other) {
     if (DEBUG) std::cerr << "test_type copy assignment" << std::endl;
-    value = other.value;
+    if (DEBUG && value_ptr == nullptr) std::cerr << "nullptr" << std::endl;
+    *value_ptr = *other.value_ptr;
     return *this;
   }
-  test_type& operator=(test_type&& other) {
+  test_type_alloc& operator=(test_type_alloc&& other) {
     if (DEBUG) std::cerr << "test_type move assignment" << std::endl;
-    value = other.value;
+    if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+    std::swap(value_ptr, other.value_ptr);
     return *this;
   }
-  int get_value() const { return value; }
+  int get_value() const {
+    if (value_ptr == nullptr) std::cerr << "null" << std::endl;
+    return *value_ptr;
+  }
 private:
-  int value;
+  int* value_ptr;
 };
 
+using test_type = test_type_alloc<std::allocator<int>>;
+
 struct test_type_hash {
   std::size_t operator()(const test_type& a) const {
     return std::hash<int>()(a.get_value());
@@ -95,7 +108,8 @@ struct test_type_serde {
     const size_t bytes_written = sizeof(int) * num;
     check_memory_size(bytes_written, capacity);
     for (unsigned i = 0; i < num; ++i) {
-      memcpy(ptr, &items[i], sizeof(int));
+      const int value = items[i].get_value();
+      memcpy(ptr, &value, sizeof(int));
       ptr = static_cast<char*>(ptr) + sizeof(int);
     }
     return bytes_written;
@@ -104,7 +118,9 @@ struct test_type_serde {
     const size_t bytes_read = sizeof(int) * num;
     check_memory_size(bytes_read, capacity);
     for (unsigned i = 0; i < num; ++i) {
-      memcpy(&items[i], ptr, sizeof(int));
+      int value;
+      memcpy(&value, ptr, sizeof(int));
+      new (&items[i]) test_type(value);
       ptr = static_cast<const char*>(ptr) + sizeof(int);
     }
     return bytes_read;
diff --git a/tuple/test/tuple_sketch_allocation_test.cpp b/tuple/test/tuple_sketch_allocation_test.cpp
index 42a1f3e..4858d3a 100644
--- a/tuple/test/tuple_sketch_allocation_test.cpp
+++ b/tuple/test/tuple_sketch_allocation_test.cpp
@@ -22,59 +22,76 @@
 #include <catch.hpp>
 #include <tuple_sketch.hpp>
 #include <test_allocator.hpp>
+#include <test_type.hpp>
 
 namespace datasketches {
 
-using update_tuple_sketch_int_alloc =
-    update_tuple_sketch<int, int, default_update_policy<int, int>, test_allocator<int>>;
-using compact_tuple_sketch_int_alloc =
-    compact_tuple_sketch<int, test_allocator<int>>;
+struct test_type_replace_policy {
+  test_type create() const { return test_type(0); }
+  void update(test_type& summary, const test_type& update) const {
+    //std::cerr << "policy::update lvalue begin" << std::endl;
+    summary = update;
+    //std::cerr << "policy::update lvalue end" << std::endl;
+  }
+  void update(test_type& summary, test_type&& update) const {
+    //std::cerr << "policy::update rvalue begin" << std::endl;
+    summary = std::move(update);
+    //std::cerr << "policy::update rvalue end" << std::endl;
+  }
+};
 
-TEST_CASE("tuple sketch with test allocator: exact mode", "[tuple_sketch]") {
+using update_tuple_sketch_test =
+    update_tuple_sketch<test_type, test_type, test_type_replace_policy, test_allocator<test_type>>;
+using compact_tuple_sketch_test =
+    compact_tuple_sketch<test_type, test_allocator<test_type>>;
+
+TEST_CASE("tuple sketch with test allocator: estimation mode", "[tuple_sketch]") {
   test_allocator_total_bytes = 0;
   test_allocator_net_allocations = 0;
   {
-    auto update_sketch = update_tuple_sketch_int_alloc::builder().build();
-    for (int i = 0; i < 10000; ++i) update_sketch.update(i, 1);
+    auto update_sketch = update_tuple_sketch_test::builder().build();
     for (int i = 0; i < 10000; ++i) update_sketch.update(i, 1);
-    REQUIRE(!update_sketch.is_empty());
-    REQUIRE(update_sketch.is_estimation_mode());
-    unsigned count = 0;
-    for (const auto& entry: update_sketch) {
-      REQUIRE(entry.second == 2);
-      ++count;
-    }
-    REQUIRE(count == update_sketch.get_num_retained());
+//    for (int i = 0; i < 10000; ++i) update_sketch.update(i, 2);
+//    REQUIRE(!update_sketch.is_empty());
+//    REQUIRE(update_sketch.is_estimation_mode());
+//    unsigned count = 0;
+//    for (const auto& entry: update_sketch) {
+//      REQUIRE(entry.second.get_value() == 2);
+//      ++count;
+//    }
+//    REQUIRE(count == update_sketch.get_num_retained());
 
-    update_sketch.trim();
-    REQUIRE(update_sketch.get_num_retained() == (1 << update_sketch.get_lg_k()));
+//    update_sketch.trim();
+//    REQUIRE(update_sketch.get_num_retained() == (1 << update_sketch.get_lg_k()));
 
-    auto compact_sketch = update_sketch.compact();
-    REQUIRE(!compact_sketch.is_empty());
-    REQUIRE(compact_sketch.is_estimation_mode());
-    count = 0;
-    for (const auto& entry: compact_sketch) {
-      REQUIRE(entry.second == 2);
-      ++count;
-    }
-    REQUIRE(count == update_sketch.get_num_retained());
+//    auto compact_sketch = update_sketch.compact();
+//    REQUIRE(!compact_sketch.is_empty());
+//    REQUIRE(compact_sketch.is_estimation_mode());
+//    count = 0;
+//    for (const auto& entry: compact_sketch) {
+//      REQUIRE(entry.second.get_value() == 2);
+//      ++count;
+//    }
+//    REQUIRE(count == update_sketch.get_num_retained());
 
-    auto bytes = compact_sketch.serialize();
-    auto deserialized_sketch = compact_tuple_sketch_int_alloc::deserialize(bytes.data(), bytes.size());
-    REQUIRE(deserialized_sketch.get_estimate() == compact_sketch.get_estimate());
+//    auto bytes = compact_sketch.serialize(0, test_type_serde());
+//    auto deserialized_sketch = compact_tuple_sketch_test::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, test_type_serde());
+//    REQUIRE(deserialized_sketch.get_estimate() == compact_sketch.get_estimate());
 
     // update sketch copy
-    update_tuple_sketch_int_alloc update_sketch_copy(update_sketch);
-    update_sketch_copy = update_sketch;
+//    std::cout << update_sketch.to_string();
+//    update_tuple_sketch_test update_sketch_copy(update_sketch);
+//    update_sketch_copy = update_sketch;
     // update sketch move
-    update_tuple_sketch_int_alloc update_sketch_moved(std::move(update_sketch_copy));
-    update_sketch_moved = std::move(update_sketch);
+//    update_tuple_sketch_test update_sketch_moved(std::move(update_sketch_copy));
+//    update_sketch_moved = std::move(update_sketch);
 
     // compact sketch copy
-    compact_tuple_sketch_int_alloc compact_sketch_copy(compact_sketch);
-    compact_sketch_copy = compact_sketch;
-    compact_tuple_sketch_int_alloc compact_sketch_moved(std::move(compact_sketch_copy));
-    compact_sketch_moved = std::move(compact_sketch);
+//    compact_tuple_sketch_test compact_sketch_copy(compact_sketch);
+//    compact_sketch_copy = compact_sketch;
+    // compact sketch move
+//    compact_tuple_sketch_test compact_sketch_moved(std::move(compact_sketch_copy));
+//    compact_sketch_moved = std::move(compact_sketch);
   }
   REQUIRE(test_allocator_total_bytes == 0);
   REQUIRE(test_allocator_net_allocations == 0);
diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp
index b4773aa..11300a3 100644
--- a/tuple/test/tuple_sketch_test.cpp
+++ b/tuple/test/tuple_sketch_test.cpp
@@ -34,7 +34,7 @@ std::ostream& operator<<(std::ostream& os, const three_doubles& tuple) {
 
 #include <catch.hpp>
 #include <tuple_sketch.hpp>
-#include <test_type.hpp>
+//#include <test_type.hpp>
 
 namespace datasketches {
 
@@ -122,8 +122,8 @@ TEST_CASE("tuple sketch float: exact mode", "[tuple_sketch]") {
     REQUIRE(deserialized_sketch.get_theta() == 1);
     REQUIRE(deserialized_sketch.get_num_retained() == 2);
     REQUIRE(deserialized_sketch.is_ordered());
-    std::cout << "deserialized sketch:" << std::endl;
-    std::cout << deserialized_sketch.to_string(true);
+//    std::cout << "deserialized sketch:" << std::endl;
+//    std::cout << deserialized_sketch.to_string(true);
   }
   { // bytes
     auto bytes = compact_sketch.serialize();
@@ -170,42 +170,6 @@ TEST_CASE("tuple sketch: float, custom policy", "[tuple_sketch]") {
   REQUIRE(sum == 22); // 5 + 10 + 7
 }
 
-struct test_type_replace_policy {
-  test_type create() const { return test_type(0); }
-  void update(test_type& summary, const test_type& update) const {
-    //std::cerr << "policy::update lvalue begin" << std::endl;
-    summary = update;
-    //std::cerr << "policy::update lvalue end" << std::endl;
-  }
-  void update(test_type& summary, test_type&& update) const {
-    //std::cerr << "policy::update rvalue begin" << std::endl;
-    summary = std::move(update);
-    //std::cerr << "policy::update rvalue end" << std::endl;
-  }
-};
-
-TEST_CASE("tuple sketch: test type with replace policy", "[tuple_sketch]") {
-  auto update_sketch = update_tuple_sketch<test_type, test_type, test_type_replace_policy>::builder().build();
-  test_type a(1);
-  update_sketch.update(1, a); // this should copy
-  update_sketch.update(2, 2); // this should move
-  update_sketch.update(1, 2); // this should move
-//  std::cout << sketch.to_string(true);
-  REQUIRE(update_sketch.get_num_retained() == 2);
-  for (const auto& entry: update_sketch) {
-    REQUIRE(entry.second.get_value() == 2);
-  }
-
-  auto compact_sketch = update_sketch.compact();
-  auto bytes = compact_sketch.serialize(0, test_type_serde());
-  auto deserialized_sketch = compact_tuple_sketch<test_type>::deserialize(bytes.data(), bytes.size(),
-      DEFAULT_SEED, test_type_serde());
-  REQUIRE(deserialized_sketch.get_num_retained() == 2);
-  for (const auto& entry: deserialized_sketch) {
-    REQUIRE(entry.second.get_value() == 2);
-  }
-}
-
 struct three_doubles_update_policy {
   std::tuple<double, double, double> create() const {
     return std::tuple<double, double, double>(0, 0, 0);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org