You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2020/11/25 20:53:12 UTC
[incubator-datasketches-cpp] branch req_sketch updated (fe6b7d5 ->
e3c6eb6)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch req_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git.
from fe6b7d5 performance improvement
new 508e325 added checking
new e3c6eb6 avoid warning
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
req/include/req_compactor_impl.hpp | 2 +-
req/include/req_sketch.hpp | 4 ++++
req/include/req_sketch_impl.hpp | 37 ++++++++++++++++++++++++++++++++++---
3 files changed, 39 insertions(+), 4 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 01/02: added checking
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch req_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit 508e3257bd73d2f725a680beec25ac232d3e05df
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Nov 25 12:52:42 2020 -0800
added checking
---
req/include/req_sketch.hpp | 4 ++++
req/include/req_sketch_impl.hpp | 37 ++++++++++++++++++++++++++++++++++---
2 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp
index aa17b2e..666e2ba 100755
--- a/req/include/req_sketch.hpp
+++ b/req/include/req_sketch.hpp
@@ -190,6 +190,10 @@ private:
class item_deleter;
req_sketch(uint32_t k, uint64_t n, std::unique_ptr<T, item_deleter> min_value, std::unique_ptr<T, item_deleter> max_value, std::vector<Compactor, AllocCompactor>&& compactors);
+ static void check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels);
+ static void check_serial_version(uint8_t serial_version);
+ static void check_family_id(uint8_t family_id);
+
// implementations for floating point types
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
static const TT& get_invalid_value() {
diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp
index 60934c8..ca72ebe 100755
--- a/req/include/req_sketch_impl.hpp
+++ b/req/include/req_sketch_impl.hpp
@@ -21,6 +21,7 @@
#define REQ_SKETCH_IMPL_HPP_
#include <sstream>
+#include <stdexcept>
namespace datasketches {
@@ -330,7 +331,9 @@ req_sketch<T, H, C, S, A> req_sketch<T, H, C, S, A>::deserialize(std::istream& i
const auto num_levels = read<uint8_t>(is);
const auto num_raw_items = read<uint8_t>(is);
- // TODO: checks
+ check_preamble_ints(preamble_ints, num_levels);
+ check_serial_version(serial_version);
+ check_family_id(family_id);
if (!is.good()) throw std::runtime_error("error reading from std::istream");
const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
@@ -409,8 +412,9 @@ req_sketch<T, H, C, S, A> req_sketch<T, H, C, S, A>::deserialize(const void* byt
uint8_t num_raw_items;
ptr += copy_from_mem(ptr, num_raw_items);
- // TODO: checks
- // ensure memory size
+ check_preamble_ints(preamble_ints, num_levels);
+ check_serial_version(serial_version);
+ check_family_id(family_id);
const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
if (is_empty) return req_sketch(k, allocator);
@@ -428,6 +432,7 @@ req_sketch<T, H, C, S, A> req_sketch<T, H, C, S, A>::deserialize(const void* byt
uint64_t n = 1;
if (num_levels > 1) {
+ ensure_minimum_memory(end_ptr - ptr, sizeof(n));
ptr += copy_from_mem(ptr, n);
ptr += S().deserialize(ptr, end_ptr - ptr, min_value_buffer.get(), 1);
// serde call did not throw, repackage with destrtuctor
@@ -584,6 +589,32 @@ max_value_(max_value.release())
update_num_retained();
}
+template<typename T, bool H, typename C, typename S, typename A>
+void req_sketch<T, H, C, S, A>::check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels) {
+ const uint8_t expected_preamble_ints = num_levels > 1 ? 4 : 2;
+ if (preamble_ints != expected_preamble_ints) {
+ throw std::invalid_argument("Possible corruption: preamble ints must be "
+ + std::to_string(expected_preamble_ints) + ", got " + std::to_string(preamble_ints));
+ }
+}
+
+template<typename T, bool H, typename C, typename S, typename A>
+void req_sketch<T, H, C, S, A>::check_serial_version(uint8_t serial_version) {
+ if (serial_version != SERIAL_VERSION) {
+ throw std::invalid_argument("Possible corruption: serial version mismatch: expected "
+ + std::to_string(SERIAL_VERSION)
+ + ", got " + std::to_string(serial_version));
+ }
+}
+
+template<typename T, bool H, typename C, typename S, typename A>
+void req_sketch<T, H, C, S, A>::check_family_id(uint8_t family_id) {
+ if (family_id != FAMILY) {
+ throw std::invalid_argument("Possible corruption: family mismatch: expected "
+ + std::to_string(FAMILY) + ", got " + std::to_string(family_id));
+ }
+}
+
} /* namespace datasketches */
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-cpp] 02/02: avoid warning
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch req_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
commit e3c6eb6789d12766f362eeca4e92a2c09ab24f42
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Nov 25 12:53:04 2020 -0800
avoid warning
---
req/include/req_compactor_impl.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp
index 462b9b8..361fa1f 100755
--- a/req/include/req_compactor_impl.hpp
+++ b/req/include/req_compactor_impl.hpp
@@ -158,7 +158,7 @@ template<typename T, bool H, typename C, typename A>
bool req_compactor<T, H, C, A>::ensure_enough_sections() {
const float ssr = section_size_raw_ / sqrt(2);
const uint32_t ne = nearest_even(ssr);
- if (state_ >= 1 << (num_sections_ - 1) && ne >= req_constants::MIN_K) {
+ if (state_ >= static_cast<uint64_t>(1 << (num_sections_ - 1)) && ne >= req_constants::MIN_K) {
section_size_raw_ = ssr;
section_size_ = ne;
num_sections_ <<= 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org