You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2019/11/05 02:03:51 UTC
[incubator-datasketches-cpp] branch cpc_template updated: common
type definitions
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch cpc_template
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-cpp.git
The following commit(s) were added to refs/heads/cpc_template by this push:
new e91b8bc common type definitions
e91b8bc is described below
commit e91b8bc19bd7902b8f96c0432b06cd5ec4a25986
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Nov 4 18:03:34 2019 -0800
common type definitions
---
cpc/include/cpc_common.hpp | 6 +++---
cpc/include/cpc_compressor.hpp | 7 +------
cpc/include/cpc_compressor_impl.hpp | 36 ++++++++++++++++++------------------
cpc/include/cpc_sketch.hpp | 9 ++-------
cpc/include/cpc_sketch_impl.hpp | 14 +++++++-------
5 files changed, 31 insertions(+), 41 deletions(-)
diff --git a/cpc/include/cpc_common.hpp b/cpc/include/cpc_common.hpp
index 4819c66..e372136 100644
--- a/cpc/include/cpc_common.hpp
+++ b/cpc/include/cpc_common.hpp
@@ -36,6 +36,8 @@ typedef std::unique_ptr<void, std::function<void(void*)>> void_ptr_with_deleter;
typedef std::unique_ptr<uint32_t, std::function<void(uint32_t*)>> u32_ptr_with_deleter;
template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
+template<typename A> using AllocU16 = typename std::allocator_traits<A>::template rebind_alloc<uint16_t>;
+template<typename A> using AllocU32 = typename std::allocator_traits<A>::template rebind_alloc<uint32_t>;
template<typename A> using AllocU64 = typename std::allocator_traits<A>::template rebind_alloc<uint64_t>;
template<typename A> using vector_u8 = std::vector<uint8_t, AllocU8<A>>;
@@ -51,10 +53,8 @@ struct compressed_state {
template<typename A>
struct uncompressed_state {
- //typedef typename std::allocator_traits<A>::template rebind_alloc<uint8_t> AllocU8;
- typedef typename std::vector<uint8_t, AllocU8<A>> window_type;
u32_table<A> table;
- window_type window;
+ vector_u8<A> window;
};
} /* namespace datasketches */
diff --git a/cpc/include/cpc_compressor.hpp b/cpc/include/cpc_compressor.hpp
index e7c1297..6c02ec6 100644
--- a/cpc/include/cpc_compressor.hpp
+++ b/cpc/include/cpc_compressor.hpp
@@ -113,11 +113,6 @@ private:
void uncompress_pinned_flavor(const compressed_state& source, uncompressed_state<A>& target, uint8_t lg_k, uint32_t num_coupons) const;
void uncompress_sliding_flavor(const compressed_state& source, uncompressed_state<A>& target, uint8_t lg_k, uint32_t num_coupons) const;
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint8_t> AllocU8;
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint16_t> AllocU16;
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint32_t> AllocU32;
- typedef typename std::vector<uint8_t, AllocU8> window_type;
-
uint8_t* make_inverse_permutation(const uint8_t* permu, int length);
uint16_t* make_decoding_table(const uint16_t* encoding_table, int num_byte_values);
void validate_decoding_table(const uint16_t* decoding_table, const uint16_t* encoding_table) const;
@@ -126,7 +121,7 @@ private:
void compress_sliding_window(const uint8_t* window, uint8_t lg_k, uint32_t num_coupons, compressed_state* target) const;
uint32_t* uncompress_surprising_values(const uint32_t* data, size_t data_words, size_t num_pairs, uint8_t lg_k) const;
- void uncompress_sliding_window(const uint32_t* data, size_t data_words, window_type& window, uint8_t lg_k, uint32_t num_coupons) const;
+ void uncompress_sliding_window(const uint32_t* data, size_t data_words, vector_u8<A>& window, uint8_t lg_k, uint32_t num_coupons) const;
static size_t safe_length_for_compressed_pair_buf(uint64_t k, size_t num_pairs, size_t num_base_bits);
static size_t safe_length_for_compressed_window_buf(uint64_t k);
diff --git a/cpc/include/cpc_compressor_impl.hpp b/cpc/include/cpc_compressor_impl.hpp
index cff9692..2a8bba2 100644
--- a/cpc/include/cpc_compressor_impl.hpp
+++ b/cpc/include/cpc_compressor_impl.hpp
@@ -52,7 +52,7 @@ cpc_compressor<A>::~cpc_compressor() {
template<typename A>
uint8_t* cpc_compressor<A>::make_inverse_permutation(const uint8_t* permu, int length) {
- uint8_t* inverse = AllocU8().allocate(length);
+ uint8_t* inverse = AllocU8<A>().allocate(length);
for (int i = 0; i < length; i++) {
inverse[permu[i]] = i;
}
@@ -67,7 +67,7 @@ uint8_t* cpc_compressor<A>::make_inverse_permutation(const uint8_t* permu, int l
// The second argument is typically 256, but can be other values such as 65.
template<typename A>
uint16_t* cpc_compressor<A>::make_decoding_table(const uint16_t* encoding_table, int num_byte_values) {
- uint16_t* decoding_table = AllocU16().allocate(4096);
+ uint16_t* decoding_table = AllocU16<A>().allocate(4096);
for (int byte_value = 0; byte_value < num_byte_values; byte_value++) {
const int encoding_entry = encoding_table[byte_value];
const int code_value = encoding_entry & 0xfff;
@@ -190,7 +190,7 @@ void cpc_compressor<A>::compress_sparse_flavor(const cpc_sketch_alloc<A>& source
const size_t num_pairs = source.surprising_value_table.get_num_items();
std::sort(pairs, &pairs[num_pairs]);
compress_surprising_values(pairs, num_pairs, source.get_lg_k(), result);
- AllocU32().deallocate(pairs, num_pairs);
+ AllocU32<A>().deallocate(pairs, num_pairs);
}
template<typename A>
@@ -199,7 +199,7 @@ void cpc_compressor<A>::uncompress_sparse_flavor(const compressed_state& source,
if (source.table_data_ptr == nullptr) throw std::logic_error("table is expected");
uint32_t* pairs = uncompress_surprising_values(source.table_data_ptr.get(), source.table_data_words, source.table_num_entries, lg_k);
target.table = u32_table<A>::make_from_pairs(pairs, source.table_num_entries, lg_k);
- AllocU32().deallocate(pairs, source.table_num_entries);
+ AllocU32<A>().deallocate(pairs, source.table_num_entries);
}
// This is complicated because it effectively builds a Sparse version
@@ -223,8 +223,8 @@ void cpc_compressor<A>::compress_hybrid_flavor(const cpc_sketch_alloc<A>& source
); // note the overlapping subarray trick
compress_surprising_values(all_pairs, source.get_num_coupons(), source.get_lg_k(), result);
- if (pairs_from_table != nullptr) AllocU32().deallocate(pairs_from_table, num_pairs_from_table);
- AllocU32().deallocate(all_pairs, source.get_num_coupons());
+ if (pairs_from_table != nullptr) AllocU32<A>().deallocate(pairs_from_table, num_pairs_from_table);
+ AllocU32<A>().deallocate(all_pairs, source.get_num_coupons());
}
template<typename A>
@@ -251,7 +251,7 @@ void cpc_compressor<A>::uncompress_hybrid_flavor(const compressed_state& source,
}
}
target.table = u32_table<A>::make_from_pairs(pairs, next_true_pair, lg_k);
- AllocU32().deallocate(pairs, source.table_num_entries);
+ AllocU32<A>().deallocate(pairs, source.table_num_entries);
}
template<typename A>
@@ -273,7 +273,7 @@ void cpc_compressor<A>::compress_pinned_flavor(const cpc_sketch_alloc<A>& source
std::sort(pairs, &pairs[num_pairs]);
compress_surprising_values(pairs, num_pairs, source.get_lg_k(), result);
- AllocU32().deallocate(pairs, num_pairs);
+ AllocU32<A>().deallocate(pairs, num_pairs);
}
}
@@ -291,7 +291,7 @@ void cpc_compressor<A>::uncompress_pinned_flavor(const compressed_state& source,
pairs[i] += 8;
}
target.table = u32_table<A>::make_from_pairs(pairs, num_pairs, lg_k);
- AllocU32().deallocate(pairs, num_pairs);
+ AllocU32<A>().deallocate(pairs, num_pairs);
}
}
@@ -325,7 +325,7 @@ void cpc_compressor<A>::compress_sliding_flavor(const cpc_sketch_alloc<A>& sourc
std::sort(pairs, &pairs[num_pairs]);
compress_surprising_values(pairs, num_pairs, source.get_lg_k(), result);
- AllocU32().deallocate(pairs, num_pairs);
+ AllocU32<A>().deallocate(pairs, num_pairs);
}
}
@@ -359,7 +359,7 @@ void cpc_compressor<A>::uncompress_sliding_flavor(const compressed_state& source
}
target.table = u32_table<A>::make_from_pairs(pairs, num_pairs, lg_k);
- AllocU32().deallocate(pairs, num_pairs);
+ AllocU32<A>().deallocate(pairs, num_pairs);
}
}
@@ -369,8 +369,8 @@ void cpc_compressor<A>::compress_surprising_values(const uint32_t* pairs, size_t
const uint64_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs);
const uint64_t table_len = safe_length_for_compressed_pair_buf(k, num_pairs, num_base_bits);
result->table_data_ptr = u32_ptr_with_deleter(
- AllocU32().allocate(table_len),
- [table_len] (uint32_t* ptr) { AllocU32().deallocate(ptr, table_len); }
+ AllocU32<A>().allocate(table_len),
+ [table_len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, table_len); }
);
size_t csv_length = low_level_compress_pairs(pairs, num_pairs, num_base_bits, result->table_data_ptr.get());
@@ -386,7 +386,7 @@ void cpc_compressor<A>::compress_surprising_values(const uint32_t* pairs, size_t
template<typename A>
uint32_t* cpc_compressor<A>::uncompress_surprising_values(const uint32_t* data, size_t data_words, size_t num_pairs, uint8_t lg_k) const {
const size_t k = 1 << lg_k;
- uint32_t* pairs = AllocU32().allocate(num_pairs);
+ uint32_t* pairs = AllocU32<A>().allocate(num_pairs);
const uint8_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs);
low_level_uncompress_pairs(pairs, num_pairs, num_base_bits, data, data_words);
return pairs;
@@ -397,8 +397,8 @@ void cpc_compressor<A>::compress_sliding_window(const uint8_t* window, uint8_t l
const size_t k = 1 << lg_k;
const size_t window_buf_len = safe_length_for_compressed_window_buf(k);
target->window_data_ptr = u32_ptr_with_deleter(
- AllocU32().allocate(window_buf_len),
- [window_buf_len] (uint32_t* ptr) { AllocU32().deallocate(ptr, window_buf_len); }
+ AllocU32<A>().allocate(window_buf_len),
+ [window_buf_len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, window_buf_len); }
);
const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons);
size_t data_words = low_level_compress_bytes(window, k, encoding_tables_for_high_entropy_byte[pseudo_phase], target->window_data_ptr.get());
@@ -411,7 +411,7 @@ void cpc_compressor<A>::compress_sliding_window(const uint8_t* window, uint8_t l
}
template<typename A>
-void cpc_compressor<A>::uncompress_sliding_window(const uint32_t* data, size_t data_words, window_type& window, uint8_t lg_k, uint32_t num_coupons) const {
+void cpc_compressor<A>::uncompress_sliding_window(const uint32_t* data, size_t data_words, vector_u8<A>& window, uint8_t lg_k, uint32_t num_coupons) const {
const size_t k = 1 << lg_k;
window.resize(k); // zeroing not needed here (unlike the Hybrid Flavor)
const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons);
@@ -736,7 +736,7 @@ void write_unary(
template<typename A>
uint32_t* cpc_compressor<A>::tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get, uint32_t empty_space) {
const size_t output_length = empty_space + num_pairs_to_get;
- uint32_t* pairs = AllocU32().allocate(output_length);
+ uint32_t* pairs = AllocU32<A>().allocate(output_length);
size_t pair_index = empty_space;
for (unsigned row_index = 0; row_index < k; row_index++) {
uint8_t byte = window[row_index];
diff --git a/cpc/include/cpc_sketch.hpp b/cpc/include/cpc_sketch.hpp
index 7fe3f46..6cc1120 100644
--- a/cpc/include/cpc_sketch.hpp
+++ b/cpc/include/cpc_sketch.hpp
@@ -97,11 +97,6 @@ class cpc_sketch_alloc {
bool validate() const;
private:
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint8_t> AllocU8;
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint32_t> AllocU32;
- typedef typename std::allocator_traits<A>::template rebind_alloc<uint64_t> AllocU64;
- typedef typename std::vector<uint8_t, AllocU8> window_type;
-
static const uint8_t SERIAL_VERSION = 1;
static const uint8_t FAMILY = 16;
@@ -124,7 +119,7 @@ class cpc_sketch_alloc {
uint32_t num_coupons; // the number of coupons collected so far
u32_table<A> surprising_value_table;
- window_type sliding_window;
+ vector_u8<A> sliding_window;
uint8_t window_offset; // derivable from num_coupons, but made explicit for speed
uint8_t first_interesting_column; // This is part of a speed optimization
@@ -133,7 +128,7 @@ class cpc_sketch_alloc {
// for deserialization and cpc_union::get_result()
cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column, u32_table<A>&& table,
- window_type&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed);
+ vector_u8<A>&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed);
inline void row_col_update(uint32_t row_col);
inline void update_sparse(uint32_t row_col);
diff --git a/cpc/include/cpc_sketch_impl.hpp b/cpc/include/cpc_sketch_impl.hpp
index b83c3d4..5f1fc30 100644
--- a/cpc/include/cpc_sketch_impl.hpp
+++ b/cpc/include/cpc_sketch_impl.hpp
@@ -454,8 +454,8 @@ std::pair<void_ptr_with_deleter, const size_t> cpc_sketch_alloc<A>::serialize(un
const uint8_t preamble_ints = get_preamble_ints(num_coupons, has_hip, has_table, has_window);
const size_t size = header_size_bytes + (preamble_ints + compressed.table_data_words + compressed.window_data_words) * sizeof(uint32_t);
void_ptr_with_deleter data_ptr(
- AllocU8().allocate(size),
- [size](void* ptr) { AllocU8().deallocate(static_cast<uint8_t*>(ptr), size); }
+ AllocU8<A>().allocate(size),
+ [size](void* ptr) { AllocU8<A>().deallocate(static_cast<uint8_t*>(ptr), size); }
);
char* ptr = static_cast<char*>(data_ptr.get()) + header_size_bytes;
ptr += copy_to_mem(ptr, &preamble_ints, sizeof(preamble_ints));
@@ -549,12 +549,12 @@ cpc_sketch_alloc<A> cpc_sketch_alloc<A>::deserialize(std::istream& is, uint64_t
}
if (has_window) {
const size_t len = compressed.window_data_words;
- compressed.window_data_ptr = u32_ptr_with_deleter(AllocU32().allocate(len), [len] (uint32_t* ptr) { AllocU32().deallocate(ptr, len); });
+ compressed.window_data_ptr = u32_ptr_with_deleter(AllocU32<A>().allocate(len), [len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, len); });
is.read((char*)compressed.window_data_ptr.get(), compressed.window_data_words * sizeof(uint32_t));
}
if (has_table) {
const size_t len = compressed.table_data_words;
- compressed.table_data_ptr = u32_ptr_with_deleter(AllocU32().allocate(len), [len] (uint32_t* ptr) { AllocU32().deallocate(ptr, len); });
+ compressed.table_data_ptr = u32_ptr_with_deleter(AllocU32<A>().allocate(len), [len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, len); });
is.read((char*)compressed.table_data_ptr.get(), compressed.table_data_words * sizeof(uint32_t));
}
if (!has_window) compressed.table_num_entries = num_coupons;
@@ -631,12 +631,12 @@ cpc_sketch_alloc<A> cpc_sketch_alloc<A>::deserialize(const void* bytes, size_t s
}
if (has_window) {
const size_t len = compressed.window_data_words;
- compressed.window_data_ptr = u32_ptr_with_deleter(AllocU32().allocate(len), [len] (uint32_t* ptr) { AllocU32().deallocate(ptr, len); });
+ compressed.window_data_ptr = u32_ptr_with_deleter(AllocU32<A>().allocate(len), [len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, len); });
ptr += copy_from_mem(ptr, compressed.window_data_ptr.get(), compressed.window_data_words * sizeof(uint32_t));
}
if (has_table) {
const size_t len = compressed.table_data_words;
- compressed.table_data_ptr = u32_ptr_with_deleter(AllocU32().allocate(len), [len] (uint32_t* ptr) { AllocU32().deallocate(ptr, len); });
+ compressed.table_data_ptr = u32_ptr_with_deleter(AllocU32<A>().allocate(len), [len] (uint32_t* ptr) { AllocU32<A>().deallocate(ptr, len); });
ptr += copy_from_mem(ptr, compressed.table_data_ptr.get(), compressed.table_data_words * sizeof(uint32_t));
}
if (!has_window) compressed.table_num_entries = num_coupons;
@@ -680,7 +680,7 @@ bool cpc_sketch_alloc<A>::validate() const {
template<typename A>
cpc_sketch_alloc<A>::cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column,
- u32_table<A>&& table, window_type&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed):
+ u32_table<A>&& table, vector_u8<A>&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed):
lg_k(lg_k),
seed(seed),
was_merged(!has_hip),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org