You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by GitBox <gi...@apache.org> on 2021/06/23 23:53:42 UTC
[GitHub] [datasketches-cpp] AlexanderSaydakov opened a new pull request #223: wrapped compact theta sketch
AlexanderSaydakov opened a new pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov merged pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
AlexanderSaydakov merged pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] coveralls edited a comment on pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
coveralls edited a comment on pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#issuecomment-867232814
## Pull Request Test Coverage Report for [Build 987577788](https://coveralls.io/builds/41018629)
* **20** of **29** **(68.97%)** changed or added relevant lines in **1** file are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage decreased (**-0.2%**) to **92.76%**
---
| Changes Missing Coverage | Covered Lines | Changed/Added Lines | % |
| :-----|--------------|--------|---: |
| [theta/include/compact_theta_sketch_parser_impl.hpp](https://coveralls.io/builds/41018629/source?filename=theta%2Finclude%2Fcompact_theta_sketch_parser_impl.hpp#L31) | 20 | 29 | 68.97%
<!-- | **Total:** | **20** | **29** | **68.97%** | -->
| Totals | [](https://coveralls.io/builds/41018629) |
| :-- | --: |
| Change from base [Build 961626121](https://coveralls.io/builds/40789146): | -0.2% |
| Covered Lines: | 2178 |
| Relevant Lines: | 2348 |
---
##### 💛 - [Coveralls](https://coveralls.io)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a change in pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
AlexanderSaydakov commented on a change in pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#discussion_r661750344
##########
File path: theta/test/theta_a_not_b_test.cpp
##########
@@ -167,6 +167,70 @@ TEST_CASE("theta a-not-b: estimation mode half overlap", "[theta_a_not_b]") {
REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
}
+TEST_CASE("theta a-not-b: estimation mode half overlap wrapped compact", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++);
+ auto bytes_a = a.compact().serialize();
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) b.update(value++);
+ auto bytes_b = b.compact().serialize();
+
+ theta_a_not_b a_not_b;
+
+ auto result = a_not_b.compute(
+ wrapped_compact_theta_sketch::wrap(bytes_a.data(), bytes_a.size()),
+ wrapped_compact_theta_sketch::wrap(bytes_b.data(), bytes_b.size())
+ );
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+template<typename V>
+std::vector<char> foo(V&& v) {
+ std::vector<char> vv;
+ std::copy(forward_begin(v), forward_end(v), std::back_inserter(vv));
+ return vv;
+}
+
+class vec {
+public:
+ vec(std::vector<char>& other) {
+ for (auto c: other) v.push_back(c);
+ }
+ std::vector<char>::iterator begin() { return v.begin(); }
+ std::vector<char>::iterator end() { return v.end(); }
+private:
+ std::vector<char> v;
+};
+
+class const_vec {
+public:
+ const_vec(const char* s, size_t size): ptr(s), sz(size) {}
+ const char* begin() const { return ptr; }
+ const char* end() const { return ptr + sz; }
+private:
+ const char* ptr;
+ size_t sz;
+};
+
+const std::vector<char> clone(std::vector<char>& other) {
+ return std::vector<char>(other);
+}
+
+TEST_CASE() {
Review comment:
Sorry, this was left there accidentally after some experimenting.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
AlexanderSaydakov commented on pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#issuecomment-870829099
<img width="1427" alt="Theta union lgk16 wrapped compact" src="https://user-images.githubusercontent.com/13126686/123849824-1d37f080-d8ce-11eb-8089-a2bf3f656f40.png">
This is a comparison of union update time with compact sketches ready at hand, compact sketches with deserialize time included, and wrapped compact sketches (effectively avoiding some cost of deserialization).
We can see that performing a union of wrapped sketches is even slightly faster than using compact sketch objects ready at hand presumably because of a faster iterator over the wrapped sketches.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] jmalkin commented on pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
jmalkin commented on pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#issuecomment-871707546
👍
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] coveralls commented on pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
coveralls commented on pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#issuecomment-867232814
## Pull Request Test Coverage Report for [Build 966171202](https://coveralls.io/builds/40833795)
* **20** of **29** **(68.97%)** changed or added relevant lines in **1** file are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage decreased (**-0.2%**) to **92.76%**
---
| Changes Missing Coverage | Covered Lines | Changed/Added Lines | % |
| :-----|--------------|--------|---: |
| [theta/include/compact_theta_sketch_parser_impl.hpp](https://coveralls.io/builds/40833795/source?filename=theta%2Finclude%2Fcompact_theta_sketch_parser_impl.hpp#L31) | 20 | 29 | 68.97%
<!-- | **Total:** | **20** | **29** | **68.97%** | -->
| Totals | [](https://coveralls.io/builds/40833795) |
| :-- | --: |
| Change from base [Build 961626121](https://coveralls.io/builds/40789146): | -0.2% |
| Covered Lines: | 2178 |
| Relevant Lines: | 2348 |
---
##### 💛 - [Coveralls](https://coveralls.io)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a change in pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
AlexanderSaydakov commented on a change in pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#discussion_r661748770
##########
File path: theta/include/compact_theta_sketch_parser_impl.hpp
##########
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+#define COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+
+#include <iostream>
+#include <iomanip>
+
+namespace datasketches {
+
+template<bool dummy>
+auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed) -> compact_theta_sketch_data {
+ if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
+ + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
+ checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+ checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ uint64_t theta = theta_constants::MAX_THETA;
+ const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+ checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+ return {true, true, seed_hash, 0, theta, nullptr};
+ }
+ const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+ if (has_theta) {
+ if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
+ theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+ }
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+ return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+ }
+ const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+ const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+ const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+ const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+ if (size < expected_size_bytes)
Review comment:
I am not sure. I can imagine a larger record and a user who does not know the size of the sketch. The primary goal here was to avoid reading past the end.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] AlexanderSaydakov commented on a change in pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
AlexanderSaydakov commented on a change in pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#discussion_r661749515
##########
File path: theta/include/compact_theta_sketch_parser_impl.hpp
##########
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+#define COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+
+#include <iostream>
+#include <iomanip>
+
+namespace datasketches {
+
+template<bool dummy>
+auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed) -> compact_theta_sketch_data {
+ if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
+ + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
+ checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+ checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ uint64_t theta = theta_constants::MAX_THETA;
+ const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+ checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+ return {true, true, seed_hash, 0, theta, nullptr};
+ }
+ const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+ if (has_theta) {
+ if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
+ theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+ }
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+ return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+ }
+ const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+ const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+ const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+ const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+ if (size < expected_size_bytes)
+ throw std::invalid_argument(std::to_string(expected_size_bytes)
+ + " bytes expected, actual " + std::to_string(size) + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
Review comment:
Right. Perhaps this should be made configurable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[GitHub] [datasketches-cpp] jmalkin commented on a change in pull request #223: wrapped compact theta sketch
Posted by GitBox <gi...@apache.org>.
jmalkin commented on a change in pull request #223:
URL: https://github.com/apache/datasketches-cpp/pull/223#discussion_r661718353
##########
File path: theta/include/compact_theta_sketch_parser_impl.hpp
##########
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+#define COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+
+#include <iostream>
+#include <iomanip>
+
+namespace datasketches {
+
+template<bool dummy>
+auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed) -> compact_theta_sketch_data {
+ if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
+ + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
+ checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+ checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ uint64_t theta = theta_constants::MAX_THETA;
+ const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+ checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+ return {true, true, seed_hash, 0, theta, nullptr};
+ }
+ const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+ if (has_theta) {
+ if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
+ theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+ }
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+ return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+ }
+ const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+ const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+ const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+ const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+ if (size < expected_size_bytes)
+ throw std::invalid_argument(std::to_string(expected_size_bytes)
+ + " bytes expected, actual " + std::to_string(size) + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
Review comment:
this dump can be quite large (MBs) in some cases. not sure if we need to consider the data security risk in that it'll happily dump arbitrary amounts of memory given a pointer and size
##########
File path: theta/include/compact_theta_sketch_parser_impl.hpp
##########
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+#define COMPACT_THETA_SKETCH_PARSER_IMPL_HPP_
+
+#include <iostream>
+#include <iomanip>
+
+namespace datasketches {
+
+template<bool dummy>
+auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed) -> compact_theta_sketch_data {
+ if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
+ + ", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size));
+ checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+ checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ uint64_t theta = theta_constants::MAX_THETA;
+ const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+ checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+ return {true, true, seed_hash, 0, theta, nullptr};
+ }
+ const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+ if (has_theta) {
+ if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
+ theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+ }
+ if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+ return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+ }
+ const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+ const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+ const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+ const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+ if (size < expected_size_bytes)
Review comment:
would we want to count too large a size as an error?
##########
File path: theta/test/theta_a_not_b_test.cpp
##########
@@ -167,6 +167,70 @@ TEST_CASE("theta a-not-b: estimation mode half overlap", "[theta_a_not_b]") {
REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
}
+TEST_CASE("theta a-not-b: estimation mode half overlap wrapped compact", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++);
+ auto bytes_a = a.compact().serialize();
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) b.update(value++);
+ auto bytes_b = b.compact().serialize();
+
+ theta_a_not_b a_not_b;
+
+ auto result = a_not_b.compute(
+ wrapped_compact_theta_sketch::wrap(bytes_a.data(), bytes_a.size()),
+ wrapped_compact_theta_sketch::wrap(bytes_b.data(), bytes_b.size())
+ );
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+template<typename V>
+std::vector<char> foo(V&& v) {
+ std::vector<char> vv;
+ std::copy(forward_begin(v), forward_end(v), std::back_inserter(vv));
+ return vv;
+}
+
+class vec {
+public:
+ vec(std::vector<char>& other) {
+ for (auto c: other) v.push_back(c);
+ }
+ std::vector<char>::iterator begin() { return v.begin(); }
+ std::vector<char>::iterator end() { return v.end(); }
+private:
+ std::vector<char> v;
+};
+
+class const_vec {
+public:
+ const_vec(const char* s, size_t size): ptr(s), sz(size) {}
+ const char* begin() const { return ptr; }
+ const char* end() const { return ptr + sz; }
+private:
+ const char* ptr;
+ size_t sz;
+};
+
+const std::vector<char> clone(std::vector<char>& other) {
+ return std::vector<char>(other);
+}
+
+TEST_CASE() {
Review comment:
do we need this test case and its supporting functions?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org