You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2020/08/14 23:46:41 UTC
[incubator-datasketches-cpp] branch tuple_sketch updated (3acc466
-> 67d7203)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git.
from 3acc466 serialization to raw bytes with tests
new 99cc639 allocation fix
new 418b974 stream type change and serialization order change
new b3f5439 better allocation test
new bae2fb0 disabled print
new 6906132 avoid swapping uninitialized payloads in rebuild
new 013b0d0 renamed method
add fc2a1b7 Merge pull request #2 from apache/master
add 30437a0 Fixed vector_of_kll_sketches check for number of elements to consider the last axis
add 7e210a8 vector_of_kll_sketches: fixed 2D updates. Added unit tests for 2D and 3D updates.
add 199829b Merge pull request #161 from mdhimes/master
add 592506e better way to specify ranges
add c6fe453 Merge pull request #162 from apache/msvc_compatibility
add 0ad6212 simplify and avoid iterating past the end
add 762904b Merge pull request #163 from apache/msvc_compatibility
add e5fea0e add array input to kll sketches in python
add c66bde3 Merge pull request #164 from apache/py_kll_array_input
add 337fbd7 Issue-165: Remove c-style const casts from kll/ and common directories
add c67d92f Merge pull request #166 from gaborkaszab/const_cast
add 2946e31 Add empty build/ directory to repo, ensure it stays empty
add 55722ca Update workflows to skip creating build/, add python descriptions
add 2ab2380 Merge pull request #167 from apache/python_description
add 7cafbef rewritten quantile calculator
add 3135dc4 avoid warnings, unified style
add 608a741 removed unused include
add fa95167 Merge pull request #168 from apache/kll_merge_blocks_fix
new 67d7203 Merge branch 'master' into tuple_sketch
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build_cmake.yml | 2 +-
.github/workflows/code_coverage.yml | 2 +-
.gitignore | 1 -
build/.gitignore | 7 ++
common/include/serde.hpp | 9 +-
common/test/test_type.hpp | 44 ++++---
kll/include/kll_quantile_calculator.hpp | 37 +++---
kll/include/kll_quantile_calculator_impl.hpp | 159 ++++++++++--------------
kll/include/kll_sketch_impl.hpp | 33 ++---
python/src/cpc_wrapper.cpp | 41 +++---
python/src/fi_wrapper.cpp | 48 ++++---
python/src/hll_wrapper.cpp | 101 +++++++++------
python/src/kll_wrapper.cpp | 115 ++++++++++++++---
python/src/theta_wrapper.cpp | 75 +++++++----
python/src/vector_of_kll.cpp | 13 +-
python/src/vo_wrapper.cpp | 44 ++++---
python/tests/kll_test.py | 8 +-
python/tests/vector_of_kll_test.py | 48 +++++++
theta/include/theta_intersection_impl.hpp | 4 +-
theta/include/theta_sketch_impl.hpp | 2 +-
theta/include/theta_union_impl.hpp | 2 +-
tuple/include/theta_update_sketch_base.hpp | 1 +
tuple/include/theta_update_sketch_base_impl.hpp | 44 +++++--
tuple/include/tuple_sketch.hpp | 6 +-
tuple/include/tuple_sketch_impl.hpp | 18 ++-
tuple/test/tuple_sketch_allocation_test.cpp | 91 ++++++++------
tuple/test/tuple_sketch_test.cpp | 42 +------
tuple/test/tuple_union_test.cpp | 6 +-
28 files changed, 617 insertions(+), 386 deletions(-)
create mode 100644 build/.gitignore
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 01/07: allocation fix
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 99cc63939914b6457de6f71b84c4957f09ef16c0
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:29:33 2020 -0700
allocation fix
---
tuple/include/theta_update_sketch_base_impl.hpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index cf6b46d..1cf88fb 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -59,7 +59,7 @@ entries_(nullptr)
entries_ = allocator_.allocate(size);
for (size_t i = 0; i < size; ++i) {
if (EK()(other.entries_[i]) != 0) {
- entries_[i] = other.entries_[i];
+ new (&entries_[i]) EN(other.entries_[i]);
} else {
EK()(entries_[i]) = 0;
}
@@ -212,7 +212,7 @@ template<typename EN, typename EK, typename A>
void theta_update_sketch_base<EN, EK, A>::rebuild() {
const size_t size = 1 << lg_cur_size_;
const uint32_t pivot = (1 << lg_nom_size_) + size - num_entries_;
- std::nth_element(&entries_[0], &entries_[pivot], &entries_[size], comparator());
+ std::nth_element(entries_, entries_ + pivot, entries_ + size, comparator());
this->theta_ = EK()(entries_[pivot]);
EN* old_entries = entries_;
entries_ = allocator_.allocate(size);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 02/07: stream type change and
serialization order change
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 418b97415f19e1f15b25b6cfe36bac9d0596f3d5
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:31:35 2020 -0700
stream type change and serialization order change
---
tuple/include/tuple_sketch.hpp | 6 +++---
tuple/include/tuple_sketch_impl.hpp | 18 ++++++++----------
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/tuple/include/tuple_sketch.hpp b/tuple/include/tuple_sketch.hpp
index 7e2475a..45f0de5 100644
--- a/tuple/include/tuple_sketch.hpp
+++ b/tuple/include/tuple_sketch.hpp
@@ -142,7 +142,7 @@ public:
virtual const_iterator end() const = 0;
protected:
- virtual void print_specifics(std::ostringstream& os) const = 0;
+ virtual void print_specifics(std::basic_ostream<char>& os) const = 0;
static uint16_t get_seed_hash(uint64_t seed);
@@ -332,7 +332,7 @@ protected:
// for builder
update_tuple_sketch(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
- virtual void print_specifics(std::ostringstream& os) const;
+ virtual void print_specifics(std::basic_ostream<char>& os) const;
};
// compact sketch
@@ -455,7 +455,7 @@ protected:
bool destroy;
};
- virtual void print_specifics(std::ostringstream& os) const;
+ virtual void print_specifics(std::basic_ostream<char>& os) const;
};
diff --git a/tuple/include/tuple_sketch_impl.hpp b/tuple/include/tuple_sketch_impl.hpp
index f0c5b50..fff3b2c 100644
--- a/tuple/include/tuple_sketch_impl.hpp
+++ b/tuple/include/tuple_sketch_impl.hpp
@@ -238,7 +238,7 @@ compact_tuple_sketch<S, A> update_tuple_sketch<S, U, P, A>::compact(bool ordered
}
template<typename S, typename U, typename P, typename A>
-void update_tuple_sketch<S, U, P, A>::print_specifics(std::ostringstream& os) const {
+void update_tuple_sketch<S, U, P, A>::print_specifics(std::basic_ostream<char>& os) const {
os << " lg nominal size : " << (int) map_.lg_nom_size_ << std::endl;
os << " lg current size : " << (int) map_.lg_cur_size_ << std::endl;
os << " resize factor : " << (1 << map_.rf_) << std::endl;
@@ -397,8 +397,6 @@ auto compact_tuple_sketch<S, A>::serialize(unsigned header_size_bytes, const Ser
}
for (const auto& it: entries_) {
ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
- }
- for (const auto& it: entries_) {
ptr += sd.serialize(ptr, end_ptr - ptr, &it.second, 1);
}
}
@@ -502,14 +500,14 @@ compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(const void* b
std::vector<Entry, AllocEntry> entries(allocator);
if (!is_empty) {
entries.reserve(num_entries);
- std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
- ptr += copy_from_mem(ptr, keys.data(), keys_size_bytes);
A alloc(allocator);
- std::unique_ptr<S, deleter_of_summaries> summaries(alloc.allocate(num_entries), deleter_of_summaries(num_entries, false));
- ptr += sd.deserialize(ptr, base + size - ptr, summaries.get(), num_entries);
- summaries.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
+ std::unique_ptr<S, deleter_of_summaries> summary(alloc.allocate(1), deleter_of_summaries(1, false));
for (size_t i = 0; i < num_entries; ++i) {
- entries.push_back(Entry(keys[i], std::move(summaries.get()[i])));
+ uint64_t key;
+ ptr += copy_from_mem(ptr, &key, sizeof(key));
+ ptr += sd.deserialize(ptr, base + size - ptr, summary.get(), 1);
+ entries.push_back(Entry(key, std::move(*summary)));
+ summary->~S();
}
}
const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
@@ -537,7 +535,7 @@ auto compact_tuple_sketch<S, A>::end() const -> const_iterator {
}
template<typename S, typename A>
-void compact_tuple_sketch<S, A>::print_specifics(std::ostringstream&) const {}
+void compact_tuple_sketch<S, A>::print_specifics(std::basic_ostream<char>&) const {}
// builder
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 04/07: disabled print
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit bae2fb029391acec38595d1f080ae3c37a43595e
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:33:24 2020 -0700
disabled print
---
tuple/test/tuple_union_test.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tuple/test/tuple_union_test.cpp b/tuple/test/tuple_union_test.cpp
index 38289fd..00127ea 100644
--- a/tuple/test/tuple_union_test.cpp
+++ b/tuple/test/tuple_union_test.cpp
@@ -29,7 +29,7 @@ TEST_CASE("tuple_union float: empty", "[tuple union]") {
auto u = tuple_union<float>::builder().build();
u.update(update_sketch);
auto result = u.get_result();
- std::cout << result.to_string(true);
+// std::cout << result.to_string(true);
REQUIRE(result.is_empty());
REQUIRE(result.get_num_retained() == 0);
REQUIRE(!result.is_estimation_mode());
@@ -38,14 +38,14 @@ TEST_CASE("tuple_union float: empty", "[tuple union]") {
TEST_CASE("tuple_union float: non-empty no retained entries", "[tuple union]") {
auto update_sketch = update_tuple_sketch<float>::builder().set_p(0.001).build();
- std::cout << update_sketch.to_string();
+// std::cout << update_sketch.to_string();
update_sketch.update(1, 1);
REQUIRE(!update_sketch.is_empty());
REQUIRE(update_sketch.get_num_retained() == 0);
auto u = tuple_union<float>::builder().build();
u.update(update_sketch);
auto result = u.get_result();
- std::cout << result.to_string(true);
+// std::cout << result.to_string(true);
REQUIRE(!result.is_empty());
REQUIRE(result.get_num_retained() == 0);
REQUIRE(result.is_estimation_mode());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 07/07: Merge branch 'master' into
tuple_sketch
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 67d7203595229704e7c0575dd9b1eef1f7e87b6e
Merge: 013b0d0 fa95167
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 16:36:14 2020 -0700
Merge branch 'master' into tuple_sketch
.github/workflows/build_cmake.yml | 2 +-
.github/workflows/code_coverage.yml | 2 +-
.gitignore | 1 -
build/.gitignore | 7 ++
common/include/serde.hpp | 9 +-
kll/include/kll_quantile_calculator.hpp | 37 ++++---
kll/include/kll_quantile_calculator_impl.hpp | 159 ++++++++++++---------------
kll/include/kll_sketch_impl.hpp | 33 +++---
python/src/cpc_wrapper.cpp | 41 ++++---
python/src/fi_wrapper.cpp | 48 +++++---
python/src/hll_wrapper.cpp | 101 ++++++++++-------
python/src/kll_wrapper.cpp | 115 +++++++++++++++----
python/src/theta_wrapper.cpp | 75 ++++++++-----
python/src/vector_of_kll.cpp | 13 ++-
python/src/vo_wrapper.cpp | 44 +++++---
python/tests/kll_test.py | 8 +-
python/tests/vector_of_kll_test.py | 48 ++++++++
theta/include/theta_intersection_impl.hpp | 4 +-
theta/include/theta_sketch_impl.hpp | 2 +-
theta/include/theta_union_impl.hpp | 2 +-
20 files changed, 481 insertions(+), 270 deletions(-)
diff --cc common/include/serde.hpp
index b8231c9,d0819d8..73e0901
--- a/common/include/serde.hpp
+++ b/common/include/serde.hpp
@@@ -47,10 -47,10 +47,10 @@@ template<typename T, typename Enable =
// with LongsSketch and ItemsSketch<Long> with ArrayOfLongsSerDe in Java
template<typename T>
struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
- void serialize(std::ostream& os, const T* items, unsigned num) {
+ void serialize(std::ostream& os, const T* items, unsigned num) const {
bool failure = false;
try {
- os.write((char*)items, sizeof(T) * num);
+ os.write(reinterpret_cast<const char*>(items), sizeof(T) * num);
} catch (std::ostream::failure& e) {
failure = true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 06/07: renamed method
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 013b0d01887c966e821d8b995978e48eb4c6e063
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 09:57:27 2020 -0700
renamed method
---
tuple/include/theta_update_sketch_base.hpp | 2 +-
tuple/include/theta_update_sketch_base_impl.hpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tuple/include/theta_update_sketch_base.hpp b/tuple/include/theta_update_sketch_base.hpp
index 5501581..e65a93d 100644
--- a/tuple/include/theta_update_sketch_base.hpp
+++ b/tuple/include/theta_update_sketch_base.hpp
@@ -87,7 +87,7 @@ struct theta_update_sketch_base {
static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
static inline uint32_t get_stride(uint64_t key, uint8_t lg_size);
- static void consolidate_empty(Entry* entries, size_t size, size_t num);
+ static void consolidate_non_empty(Entry* entries, size_t size, size_t num);
};
// builder
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index 7047f98..baf94ea 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -238,7 +238,7 @@ void theta_update_sketch_base<EN, EK, A>::trim() {
}
template<typename EN, typename EK, typename A>
-void theta_update_sketch_base<EN, EK, A>::consolidate_empty(EN* entries, size_t size, size_t num) {
+void theta_update_sketch_base<EN, EK, A>::consolidate_non_empty(EN* entries, size_t size, size_t num) {
// find the first empty slot
size_t i = 0;
while (i < size) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 05/07: avoid swapping uninitialized
payloads in rebuild
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 690613213a4e8acdd62ab3355bdb716da47a28bd
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Aug 14 09:54:47 2020 -0700
avoid swapping uninitialized payloads in rebuild
---
tuple/include/theta_update_sketch_base.hpp | 1 +
tuple/include/theta_update_sketch_base_impl.hpp | 42 +++++++++++++++++++------
2 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/tuple/include/theta_update_sketch_base.hpp b/tuple/include/theta_update_sketch_base.hpp
index 15b6acd..5501581 100644
--- a/tuple/include/theta_update_sketch_base.hpp
+++ b/tuple/include/theta_update_sketch_base.hpp
@@ -87,6 +87,7 @@ struct theta_update_sketch_base {
static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
static inline uint32_t get_stride(uint64_t key, uint8_t lg_size);
+ static void consolidate_empty(Entry* entries, size_t size, size_t num);
};
// builder
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
index 1cf88fb..7047f98 100644
--- a/tuple/include/theta_update_sketch_base_impl.hpp
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -208,22 +208,26 @@ void theta_update_sketch_base<EN, EK, A>::resize() {
allocator_.deallocate(old_entries, old_size);
}
+// assumes number of entries > nominal size
template<typename EN, typename EK, typename A>
void theta_update_sketch_base<EN, EK, A>::rebuild() {
const size_t size = 1 << lg_cur_size_;
- const uint32_t pivot = (1 << lg_nom_size_) + size - num_entries_;
- std::nth_element(entries_, entries_ + pivot, entries_ + size, comparator());
- this->theta_ = EK()(entries_[pivot]);
+ const uint32_t nominal_size = 1 << lg_nom_size_;
+
+ // empty entries have uninitialized payloads
+ // TODO: avoid this for empty or trivial payloads (arithmetic types)
+ consolidate_non_empty(entries_, size, num_entries_);
+
+ std::nth_element(entries_, entries_ + nominal_size, entries_ + num_entries_, comparator());
+ this->theta_ = EK()(entries_[nominal_size]);
EN* old_entries = entries_;
entries_ = allocator_.allocate(size);
for (size_t i = 0; i < size; ++i) EK()(entries_[i]) = 0;
num_entries_ = 0;
- for (size_t i = 0; i < size; ++i) {
- const uint64_t key = EK()(old_entries[i]);
- if (key != 0 && key < this->theta_) {
- insert(find(key).first, std::move(old_entries[i])); // consider a special insert with no comparison
- old_entries[i].~EN();
- }
+ // relies on consolidating non-empty entries to the front
+ for (size_t i = 0; i < nominal_size; ++i) {
+ insert(find(EK()(old_entries[i])).first, std::move(old_entries[i])); // consider a special insert with no comparison
+ old_entries[i].~EN();
}
allocator_.deallocate(old_entries, size);
}
@@ -233,6 +237,26 @@ void theta_update_sketch_base<EN, EK, A>::trim() {
if (num_entries_ > static_cast<uint32_t>(1 << lg_nom_size_)) rebuild();
}
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::consolidate_empty(EN* entries, size_t size, size_t num) {
+ // find the first empty slot
+ size_t i = 0;
+ while (i < size) {
+ if (EK()(entries[i]) == 0) break;
+ ++i;
+ }
+ // scan the rest and move non-empty entries to the front
+ for (size_t j = i + 1; j < size; ++j) {
+ if (EK()(entries[j]) != 0) {
+ new (&entries[i]) EN(std::move(entries[j]));
+ entries[j].~EN();
+ EK()(entries[j]) = 0;
+ ++i;
+ if (i == num) break;
+ }
+ }
+}
+
// builder
template<typename Derived>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 03/07: better allocation test
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch tuple_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit b3f5439ee85e6d0e210c3fb768209dbfc26245f0
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Aug 12 15:33:04 2020 -0700
better allocation test
---
common/test/test_type.hpp | 44 +++++++++-----
tuple/test/tuple_sketch_allocation_test.cpp | 91 +++++++++++++++++------------
tuple/test/tuple_sketch_test.cpp | 42 +------------
3 files changed, 87 insertions(+), 90 deletions(-)
diff --git a/common/test/test_type.hpp b/common/test/test_type.hpp
index cbd1a2e..18be598 100644
--- a/common/test/test_type.hpp
+++ b/common/test/test_type.hpp
@@ -24,38 +24,51 @@
namespace datasketches {
-class test_type {
- static const bool DEBUG = true;
+template<typename A>
+class test_type_alloc {
+ static const bool DEBUG = false;
public:
// no default constructor should be required
- test_type(int value): value(value) {
+ test_type_alloc(int value): value_ptr(A().allocate(1)) {
if (DEBUG) std::cerr << "test_type constructor" << std::endl;
+ *value_ptr = value;
}
- ~test_type() {
+ ~test_type_alloc() {
if (DEBUG) std::cerr << "test_type destructor" << std::endl;
+ if (value_ptr != nullptr) A().deallocate(value_ptr, 1);
}
- test_type(const test_type& other): value(other.value) {
+ test_type_alloc(const test_type_alloc& other): value_ptr(A().allocate(1)) {
if (DEBUG) std::cerr << "test_type copy constructor" << std::endl;
+ *value_ptr = *other.value_ptr;
}
// noexcept is important here so that, for instance, std::vector could move this type
- test_type(test_type&& other) noexcept : value(other.value) {
+ test_type_alloc(test_type_alloc&& other) noexcept : value_ptr(nullptr) {
if (DEBUG) std::cerr << "test_type move constructor" << std::endl;
+ if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+ std::swap(value_ptr, other.value_ptr);
}
- test_type& operator=(const test_type& other) {
+ test_type_alloc& operator=(const test_type_alloc& other) {
if (DEBUG) std::cerr << "test_type copy assignment" << std::endl;
- value = other.value;
+ if (DEBUG && value_ptr == nullptr) std::cerr << "nullptr" << std::endl;
+ *value_ptr = *other.value_ptr;
return *this;
}
- test_type& operator=(test_type&& other) {
+ test_type_alloc& operator=(test_type_alloc&& other) {
if (DEBUG) std::cerr << "test_type move assignment" << std::endl;
- value = other.value;
+ if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+ std::swap(value_ptr, other.value_ptr);
return *this;
}
- int get_value() const { return value; }
+ int get_value() const {
+ if (value_ptr == nullptr) std::cerr << "null" << std::endl;
+ return *value_ptr;
+ }
private:
- int value;
+ int* value_ptr;
};
+using test_type = test_type_alloc<std::allocator<int>>;
+
struct test_type_hash {
std::size_t operator()(const test_type& a) const {
return std::hash<int>()(a.get_value());
@@ -95,7 +108,8 @@ struct test_type_serde {
const size_t bytes_written = sizeof(int) * num;
check_memory_size(bytes_written, capacity);
for (unsigned i = 0; i < num; ++i) {
- memcpy(ptr, &items[i], sizeof(int));
+ const int value = items[i].get_value();
+ memcpy(ptr, &value, sizeof(int));
ptr = static_cast<char*>(ptr) + sizeof(int);
}
return bytes_written;
@@ -104,7 +118,9 @@ struct test_type_serde {
const size_t bytes_read = sizeof(int) * num;
check_memory_size(bytes_read, capacity);
for (unsigned i = 0; i < num; ++i) {
- memcpy(&items[i], ptr, sizeof(int));
+ int value;
+ memcpy(&value, ptr, sizeof(int));
+ new (&items[i]) test_type(value);
ptr = static_cast<const char*>(ptr) + sizeof(int);
}
return bytes_read;
diff --git a/tuple/test/tuple_sketch_allocation_test.cpp b/tuple/test/tuple_sketch_allocation_test.cpp
index 42a1f3e..4858d3a 100644
--- a/tuple/test/tuple_sketch_allocation_test.cpp
+++ b/tuple/test/tuple_sketch_allocation_test.cpp
@@ -22,59 +22,76 @@
#include <catch.hpp>
#include <tuple_sketch.hpp>
#include <test_allocator.hpp>
+#include <test_type.hpp>
namespace datasketches {
-using update_tuple_sketch_int_alloc =
- update_tuple_sketch<int, int, default_update_policy<int, int>, test_allocator<int>>;
-using compact_tuple_sketch_int_alloc =
- compact_tuple_sketch<int, test_allocator<int>>;
+struct test_type_replace_policy {
+ test_type create() const { return test_type(0); }
+ void update(test_type& summary, const test_type& update) const {
+ //std::cerr << "policy::update lvalue begin" << std::endl;
+ summary = update;
+ //std::cerr << "policy::update lvalue end" << std::endl;
+ }
+ void update(test_type& summary, test_type&& update) const {
+ //std::cerr << "policy::update rvalue begin" << std::endl;
+ summary = std::move(update);
+ //std::cerr << "policy::update rvalue end" << std::endl;
+ }
+};
-TEST_CASE("tuple sketch with test allocator: exact mode", "[tuple_sketch]") {
+using update_tuple_sketch_test =
+ update_tuple_sketch<test_type, test_type, test_type_replace_policy, test_allocator<test_type>>;
+using compact_tuple_sketch_test =
+ compact_tuple_sketch<test_type, test_allocator<test_type>>;
+
+TEST_CASE("tuple sketch with test allocator: estimation mode", "[tuple_sketch]") {
test_allocator_total_bytes = 0;
test_allocator_net_allocations = 0;
{
- auto update_sketch = update_tuple_sketch_int_alloc::builder().build();
- for (int i = 0; i < 10000; ++i) update_sketch.update(i, 1);
+ auto update_sketch = update_tuple_sketch_test::builder().build();
for (int i = 0; i < 10000; ++i) update_sketch.update(i, 1);
- REQUIRE(!update_sketch.is_empty());
- REQUIRE(update_sketch.is_estimation_mode());
- unsigned count = 0;
- for (const auto& entry: update_sketch) {
- REQUIRE(entry.second == 2);
- ++count;
- }
- REQUIRE(count == update_sketch.get_num_retained());
+// for (int i = 0; i < 10000; ++i) update_sketch.update(i, 2);
+// REQUIRE(!update_sketch.is_empty());
+// REQUIRE(update_sketch.is_estimation_mode());
+// unsigned count = 0;
+// for (const auto& entry: update_sketch) {
+// REQUIRE(entry.second.get_value() == 2);
+// ++count;
+// }
+// REQUIRE(count == update_sketch.get_num_retained());
- update_sketch.trim();
- REQUIRE(update_sketch.get_num_retained() == (1 << update_sketch.get_lg_k()));
+// update_sketch.trim();
+// REQUIRE(update_sketch.get_num_retained() == (1 << update_sketch.get_lg_k()));
- auto compact_sketch = update_sketch.compact();
- REQUIRE(!compact_sketch.is_empty());
- REQUIRE(compact_sketch.is_estimation_mode());
- count = 0;
- for (const auto& entry: compact_sketch) {
- REQUIRE(entry.second == 2);
- ++count;
- }
- REQUIRE(count == update_sketch.get_num_retained());
+// auto compact_sketch = update_sketch.compact();
+// REQUIRE(!compact_sketch.is_empty());
+// REQUIRE(compact_sketch.is_estimation_mode());
+// count = 0;
+// for (const auto& entry: compact_sketch) {
+// REQUIRE(entry.second.get_value() == 2);
+// ++count;
+// }
+// REQUIRE(count == update_sketch.get_num_retained());
- auto bytes = compact_sketch.serialize();
- auto deserialized_sketch = compact_tuple_sketch_int_alloc::deserialize(bytes.data(), bytes.size());
- REQUIRE(deserialized_sketch.get_estimate() == compact_sketch.get_estimate());
+// auto bytes = compact_sketch.serialize(0, test_type_serde());
+// auto deserialized_sketch = compact_tuple_sketch_test::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, test_type_serde());
+// REQUIRE(deserialized_sketch.get_estimate() == compact_sketch.get_estimate());
// update sketch copy
- update_tuple_sketch_int_alloc update_sketch_copy(update_sketch);
- update_sketch_copy = update_sketch;
+// std::cout << update_sketch.to_string();
+// update_tuple_sketch_test update_sketch_copy(update_sketch);
+// update_sketch_copy = update_sketch;
// update sketch move
- update_tuple_sketch_int_alloc update_sketch_moved(std::move(update_sketch_copy));
- update_sketch_moved = std::move(update_sketch);
+// update_tuple_sketch_test update_sketch_moved(std::move(update_sketch_copy));
+// update_sketch_moved = std::move(update_sketch);
// compact sketch copy
- compact_tuple_sketch_int_alloc compact_sketch_copy(compact_sketch);
- compact_sketch_copy = compact_sketch;
- compact_tuple_sketch_int_alloc compact_sketch_moved(std::move(compact_sketch_copy));
- compact_sketch_moved = std::move(compact_sketch);
+// compact_tuple_sketch_test compact_sketch_copy(compact_sketch);
+// compact_sketch_copy = compact_sketch;
+ // compact sketch move
+// compact_tuple_sketch_test compact_sketch_moved(std::move(compact_sketch_copy));
+// compact_sketch_moved = std::move(compact_sketch);
}
REQUIRE(test_allocator_total_bytes == 0);
REQUIRE(test_allocator_net_allocations == 0);
diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp
index b4773aa..11300a3 100644
--- a/tuple/test/tuple_sketch_test.cpp
+++ b/tuple/test/tuple_sketch_test.cpp
@@ -34,7 +34,7 @@ std::ostream& operator<<(std::ostream& os, const three_doubles& tuple) {
#include <catch.hpp>
#include <tuple_sketch.hpp>
-#include <test_type.hpp>
+//#include <test_type.hpp>
namespace datasketches {
@@ -122,8 +122,8 @@ TEST_CASE("tuple sketch float: exact mode", "[tuple_sketch]") {
REQUIRE(deserialized_sketch.get_theta() == 1);
REQUIRE(deserialized_sketch.get_num_retained() == 2);
REQUIRE(deserialized_sketch.is_ordered());
- std::cout << "deserialized sketch:" << std::endl;
- std::cout << deserialized_sketch.to_string(true);
+// std::cout << "deserialized sketch:" << std::endl;
+// std::cout << deserialized_sketch.to_string(true);
}
{ // bytes
auto bytes = compact_sketch.serialize();
@@ -170,42 +170,6 @@ TEST_CASE("tuple sketch: float, custom policy", "[tuple_sketch]") {
REQUIRE(sum == 22); // 5 + 10 + 7
}
-struct test_type_replace_policy {
- test_type create() const { return test_type(0); }
- void update(test_type& summary, const test_type& update) const {
- //std::cerr << "policy::update lvalue begin" << std::endl;
- summary = update;
- //std::cerr << "policy::update lvalue end" << std::endl;
- }
- void update(test_type& summary, test_type&& update) const {
- //std::cerr << "policy::update rvalue begin" << std::endl;
- summary = std::move(update);
- //std::cerr << "policy::update rvalue end" << std::endl;
- }
-};
-
-TEST_CASE("tuple sketch: test type with replace policy", "[tuple_sketch]") {
- auto update_sketch = update_tuple_sketch<test_type, test_type, test_type_replace_policy>::builder().build();
- test_type a(1);
- update_sketch.update(1, a); // this should copy
- update_sketch.update(2, 2); // this should move
- update_sketch.update(1, 2); // this should move
-// std::cout << sketch.to_string(true);
- REQUIRE(update_sketch.get_num_retained() == 2);
- for (const auto& entry: update_sketch) {
- REQUIRE(entry.second.get_value() == 2);
- }
-
- auto compact_sketch = update_sketch.compact();
- auto bytes = compact_sketch.serialize(0, test_type_serde());
- auto deserialized_sketch = compact_tuple_sketch<test_type>::deserialize(bytes.data(), bytes.size(),
- DEFAULT_SEED, test_type_serde());
- REQUIRE(deserialized_sketch.get_num_retained() == 2);
- for (const auto& entry: deserialized_sketch) {
- REQUIRE(entry.second.get_value() == 2);
- }
-}
-
struct three_doubles_update_policy {
std::tuple<double, double, double> create() const {
return std::tuple<double, double, double>(0, 0, 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org