You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/01/10 01:19:35 UTC
[datasketches-cpp] branch theta_compression created (now c88f1a5)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch theta_compression
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
at c88f1a5 compressed theta sketch
This branch includes the following new commits:
new c88f1a5 compressed theta sketch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-cpp] 01/01: compressed theta sketch
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch theta_compression
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
commit c88f1a52265fb41ac5f82bd8005cc11f834d0ff7
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Jan 9 17:19:28 2023 -0800
compressed theta sketch
---
common/include/count_zeros.hpp | 11 +
theta/include/bit_packing.hpp | 6286 ++++++++++++++++++++
theta/include/compact_theta_sketch_parser.hpp | 23 +-
theta/include/compact_theta_sketch_parser_impl.hpp | 95 +-
theta/include/theta_sketch.hpp | 46 +-
theta/include/theta_sketch_impl.hpp | 437 +-
theta/test/theta_sketch_test.cpp | 19 +-
7 files changed, 6831 insertions(+), 86 deletions(-)
diff --git a/common/include/count_zeros.hpp b/common/include/count_zeros.hpp
index cdd9940..51cbc0c 100644
--- a/common/include/count_zeros.hpp
+++ b/common/include/count_zeros.hpp
@@ -91,6 +91,17 @@ static inline uint8_t count_leading_zeros_in_u64(uint64_t input) {
return 56 + byte_leading_zeros_table[(input ) & FCLZ_MASK_08];
}
+static inline uint8_t count_leading_zeros_in_u32(uint32_t input) {
+ if (input > FCLZ_MASK_24)
+ return byte_leading_zeros_table[(input >> 24) & FCLZ_MASK_08];
+ if (input > FCLZ_MASK_16)
+ return 8 + byte_leading_zeros_table[(input >> 16) & FCLZ_MASK_08];
+ if (input > FCLZ_MASK_08)
+ return 16 + byte_leading_zeros_table[(input >> 8) & FCLZ_MASK_08];
+ if (true)
+ return 24 + byte_leading_zeros_table[(input ) & FCLZ_MASK_08];
+}
+
static inline uint8_t count_trailing_zeros_in_u32(uint32_t input) {
for (int i = 0; i < 4; i++) {
const int byte = input & 0xff;
diff --git a/theta/include/bit_packing.hpp b/theta/include/bit_packing.hpp
new file mode 100644
index 0000000..48166e8
--- /dev/null
+++ b/theta/include/bit_packing.hpp
@@ -0,0 +1,6286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef BIT_PACKING_HPP_
+#define BIT_PACKING_HPP_
+
+#include <memory>
+
+namespace datasketches {
+
+static inline uint8_t pack_bits(uint64_t value, uint8_t width, uint8_t*& ptr, uint8_t offset) {
+ if (offset > 0) {
+ const uint8_t chunk_bits = 8 - offset;
+ const uint8_t mask = (1 << chunk_bits) - 1;
+ if (width < chunk_bits) {
+ *ptr |= (value << (chunk_bits - width)) & mask;
+ return offset + width;
+ }
+ *ptr++ |= (value >> (width - chunk_bits)) & mask;
+ width -= chunk_bits;
+ }
+ while (width >= 8) {
+ *ptr++ = value >> (width - 8);
+ width -= 8;
+ }
+ if (width > 0) {
+ *ptr = value << (8 - width);
+ return width;
+ }
+ return 0;
+}
+
+static inline uint8_t unpack_bits(uint64_t& value, uint8_t width, const uint8_t*& ptr, uint8_t offset) {
+ const uint8_t avail_bits = 8 - offset;
+ const uint8_t chunk_bits = std::min(avail_bits, width);
+ const uint8_t mask = (1 << chunk_bits) - 1;
+ value = (*ptr >> (avail_bits - chunk_bits)) & mask;
+ ptr += avail_bits == chunk_bits;
+ offset = (offset + chunk_bits) & 7;
+ width -= chunk_bits;
+ while (width >= 8) {
+ value <<= 8;
+ value |= *ptr++;
+ width -= 8;
+ }
+ if (width > 0) {
+ value <<= width;
+ value |= *ptr >> (8 - width);
+ return width;
+ }
+ return offset;
+}
+
+static inline size_t pack_ULEB128(uint64_t value, uint8_t* ptr) {
+ const uint8_t* start = ptr;
+ while (value >= 0x80) {
+ *ptr++ = value | 0x80;
+ value >>= 7;
+ }
+ *ptr++ = value;
+ return ptr - start;
+}
+
+// pack given number of bits from a block of 8 64-bit values into bytes
+
+static inline void pack_bits_1(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 7;
+ *ptr |= values[1] << 6;
+ *ptr |= values[2] << 5;
+ *ptr |= values[3] << 4;
+ *ptr |= values[4] << 3;
+ *ptr |= values[5] << 2;
+ *ptr |= values[6] << 1;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_2(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 6;
+ *ptr |= values[1] << 4;
+ *ptr |= values[2] << 2;
+ *ptr++ |= values[3];
+
+ *ptr = values[4] << 6;
+ *ptr |= values[5] << 4;
+ *ptr |= values[6] << 2;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_3(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 5;
+ *ptr |= values[1] << 2;
+ *ptr++ |= values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr |= values[3] << 4;
+ *ptr |= values[4] << 1;
+ *ptr++ |= values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr |= values[6] << 3;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_4(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1];
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3];
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5];
+
+ *ptr = values[6] << 4;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_5(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr |= values[2] << 1;
+ *ptr++ |= values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr |= values[5] << 2;
+ *ptr++ |= values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_6(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3];
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_7(const uint64_t* values, uint8_t* ptr) {
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr |= values[7];
+}
+
+static inline void pack_bits_8(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0];
+ *ptr++ = values[1];
+ *ptr++ = values[2];
+ *ptr++ = values[3];
+ *ptr++ = values[4];
+ *ptr++ = values[5];
+ *ptr++ = values[6];
+ *ptr = values[7];
+}
+
+static inline void pack_bits_9(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_10(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_11(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 9;
+
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 10;
+
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_12(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 8;
+
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 8;
+
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_13(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 10;
+
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] >> 4;
+ *ptr++ |= values[4] >> 9;
+
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 11;
+
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_14(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 12;
+
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 10;
+
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 12;
+
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 10;
+
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_15(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 14;
+
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 13;
+
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 11;
+
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 10;
+
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 9;
+
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_16(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_17(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 9;
+
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 10;
+
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 11;
+
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 13;
+
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 14;
+
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 15;
+
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_18(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 10;
+
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 12;
+
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 14;
+
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 16;
+
+ *ptr++ = values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 10;
+
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 12;
+
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 14;
+
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_19(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 11;
+
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 14;
+
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 17;
+
+ *ptr++ = values[2] >> 9;
+
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 15;
+
+ *ptr++ |= values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 18;
+
+ *ptr++ = values[5] >> 10;
+
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 13;
+
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_20(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 12;
+
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 16;
+
+ *ptr++ = values[1] >> 8;
+
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 12;
+
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 16;
+
+ *ptr++ = values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 12;
+
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 16;
+
+ *ptr++ = values[5] >> 8;
+
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 12;
+
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_21(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 13;
+
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 18;
+
+ *ptr++ = values[1] >> 10;
+
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 15;
+
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 20;
+
+ *ptr++ = values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 17;
+
+ *ptr++ = values[4] >> 9;
+
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 14;
+
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 19;
+
+ *ptr++ = values[6] >> 11;
+
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_22(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 14;
+
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 20;
+
+ *ptr++ = values[1] >> 12;
+
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 18;
+
+ *ptr++ = values[2] >> 10;
+
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 16;
+
+ *ptr++ = values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 14;
+
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 20;
+
+ *ptr++ = values[5] >> 12;
+
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 18;
+
+ *ptr++ = values[6] >> 10;
+
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_23(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 15;
+
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 22;
+
+ *ptr++ = values[1] >> 14;
+
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 21;
+
+ *ptr++ = values[2] >> 13;
+
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 20;
+
+ *ptr++ = values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 19;
+
+ *ptr++ = values[4] >> 11;
+
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 18;
+
+ *ptr++ = values[5] >> 10;
+
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 17;
+
+ *ptr++ = values[6] >> 9;
+
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_24(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 16;
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 16;
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 16;
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 16;
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_25(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 17;
+
+ *ptr++ = values[0] >> 9;
+
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 18;
+
+ *ptr++ = values[1] >> 10;
+
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 19;
+
+ *ptr++ = values[2] >> 11;
+
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 20;
+
+ *ptr++ = values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 21;
+
+ *ptr++ = values[4] >> 13;
+
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 22;
+
+ *ptr++ = values[5] >> 14;
+
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 23;
+
+ *ptr++ = values[6] >> 15;
+
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 24;
+
+ *ptr++ = values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_26(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 18;
+
+ *ptr++ = values[0] >> 10;
+
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 20;
+
+ *ptr++ = values[1] >> 12;
+
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 22;
+
+ *ptr++ = values[2] >> 14;
+
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 24;
+
+ *ptr++ = values[3] >> 16;
+
+ *ptr++ = values[3] >> 8;
+
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 18;
+
+ *ptr++ = values[4] >> 10;
+
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 20;
+
+ *ptr++ = values[5] >> 12;
+
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 22;
+
+ *ptr++ = values[6] >> 14;
+
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 24;
+
+ *ptr++ = values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_27(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 19;
+
+ *ptr++ = values[0] >> 11;
+
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 22;
+
+ *ptr++ = values[1] >> 14;
+
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 25;
+
+ *ptr++ = values[2] >> 17;
+
+ *ptr++ = values[2] >> 9;
+
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 20;
+
+ *ptr++ = values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 23;
+
+ *ptr++ = values[4] >> 15;
+
+ *ptr++ = values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 26;
+
+ *ptr++ = values[5] >> 18;
+
+ *ptr++ = values[5] >> 10;
+
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 21;
+
+ *ptr++ = values[6] >> 13;
+
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 24;
+
+ *ptr++ = values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_28(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 20;
+ *ptr++ = values[0] >> 12;
+ *ptr++ = values[0] >> 4;
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+ *ptr++ = values[2] >> 20;
+ *ptr++ = values[2] >> 12;
+ *ptr++ = values[2] >> 4;
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+ *ptr++ = values[4] >> 20;
+ *ptr++ = values[4] >> 12;
+ *ptr++ = values[4] >> 4;
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+ *ptr++ = values[6] >> 20;
+ *ptr++ = values[6] >> 12;
+ *ptr++ = values[6] >> 4;
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_29(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 21;
+
+ *ptr++ = values[0] >> 13;
+
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 26;
+
+ *ptr++ = values[1] >> 18;
+
+ *ptr++ = values[1] >> 10;
+
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 23;
+
+ *ptr++ = values[2] >> 15;
+
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 28;
+
+ *ptr++ = values[3] >> 20;
+
+ *ptr++ = values[3] >> 12;
+
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 25;
+
+ *ptr++ = values[4] >> 17;
+
+ *ptr++ = values[4] >> 9;
+
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 22;
+
+ *ptr++ = values[5] >> 14;
+
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 27;
+
+ *ptr++ = values[6] >> 19;
+
+ *ptr++ = values[6] >> 11;
+
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 24;
+
+ *ptr++ = values[7] >> 16;
+
+ *ptr++ = values[7] >> 8;
+
+ *ptr = values[7];
+}
+
+static inline void pack_bits_30(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 22;
+ *ptr++ = values[0] >> 14;
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 26;
+ *ptr++ = values[2] >> 18;
+ *ptr++ = values[2] >> 10;
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 22;
+ *ptr++ = values[4] >> 14;
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 26;
+ *ptr++ = values[6] >> 18;
+ *ptr++ = values[6] >> 10;
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_31(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 23;
+ *ptr++ = values[0] >> 15;
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 29;
+ *ptr++ = values[2] >> 21;
+ *ptr++ = values[2] >> 13;
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 27;
+ *ptr++ = values[4] >> 19;
+ *ptr++ = values[4] >> 11;
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 25;
+ *ptr++ = values[6] >> 17;
+ *ptr++ = values[6] >> 9;
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_32(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 24;
+ *ptr++ = values[0] >> 16;
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 24;
+ *ptr++ = values[2] >> 16;
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 24;
+ *ptr++ = values[4] >> 16;
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 24;
+ *ptr++ = values[6] >> 16;
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_33(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 25;
+ *ptr++ = values[0] >> 17;
+ *ptr++ = values[0] >> 9;
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 27;
+ *ptr++ = values[2] >> 19;
+ *ptr++ = values[2] >> 11;
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 29;
+ *ptr++ = values[4] >> 21;
+ *ptr++ = values[4] >> 13;
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 31;
+ *ptr++ = values[6] >> 23;
+ *ptr++ = values[6] >> 15;
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_34(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 26;
+ *ptr++ = values[0] >> 18;
+ *ptr++ = values[0] >> 10;
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 30;
+ *ptr++ = values[2] >> 22;
+ *ptr++ = values[2] >> 14;
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 26;
+ *ptr++ = values[4] >> 18;
+ *ptr++ = values[4] >> 10;
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 30;
+ *ptr++ = values[6] >> 22;
+ *ptr++ = values[6] >> 14;
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_35(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 27;
+ *ptr++ = values[0] >> 19;
+ *ptr++ = values[0] >> 11;
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 33;
+ *ptr++ = values[2] >> 25;
+ *ptr++ = values[2] >> 17;
+ *ptr++ = values[2] >> 9;
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 31;
+ *ptr++ = values[4] >> 23;
+ *ptr++ = values[4] >> 15;
+ *ptr++ = values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 29;
+ *ptr++ = values[6] >> 21;
+ *ptr++ = values[6] >> 13;
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_36(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 28;
+ *ptr++ = values[0] >> 20;
+ *ptr++ = values[0] >> 12;
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 28;
+ *ptr++ = values[2] >> 20;
+ *ptr++ = values[2] >> 12;
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 28;
+ *ptr++ = values[4] >> 20;
+ *ptr++ = values[4] >> 12;
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 28;
+ *ptr++ = values[6] >> 20;
+ *ptr++ = values[6] >> 12;
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_37(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 29;
+ *ptr++ = values[0] >> 21;
+ *ptr++ = values[0] >> 13;
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 31;
+ *ptr++ = values[2] >> 23;
+ *ptr++ = values[2] >> 15;
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 33;
+ *ptr++ = values[4] >> 25;
+ *ptr++ = values[4] >> 17;
+ *ptr++ = values[4] >> 9;
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 35;
+ *ptr++ = values[6] >> 27;
+ *ptr++ = values[6] >> 19;
+ *ptr++ = values[6] >> 11;
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_38(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 30;
+ *ptr++ = values[0] >> 22;
+ *ptr++ = values[0] >> 14;
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 34;
+ *ptr++ = values[2] >> 26;
+ *ptr++ = values[2] >> 18;
+ *ptr++ = values[2] >> 10;
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 30;
+ *ptr++ = values[4] >> 22;
+ *ptr++ = values[4] >> 14;
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 34;
+ *ptr++ = values[6] >> 26;
+ *ptr++ = values[6] >> 18;
+ *ptr++ = values[6] >> 10;
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_39(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 31;
+ *ptr++ = values[0] >> 23;
+ *ptr++ = values[0] >> 15;
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 37;
+ *ptr++ = values[2] >> 29;
+ *ptr++ = values[2] >> 21;
+ *ptr++ = values[2] >> 13;
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 35;
+ *ptr++ = values[4] >> 27;
+ *ptr++ = values[4] >> 19;
+ *ptr++ = values[4] >> 11;
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 33;
+ *ptr++ = values[6] >> 25;
+ *ptr++ = values[6] >> 17;
+ *ptr++ = values[6] >> 9;
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_40(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 32;
+ *ptr++ = values[0] >> 24;
+ *ptr++ = values[0] >> 16;
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 32;
+ *ptr++ = values[2] >> 24;
+ *ptr++ = values[2] >> 16;
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 32;
+ *ptr++ = values[4] >> 24;
+ *ptr++ = values[4] >> 16;
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 32;
+ *ptr++ = values[6] >> 24;
+ *ptr++ = values[6] >> 16;
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_41(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 33;
+ *ptr++ = values[0] >> 25;
+ *ptr++ = values[0] >> 17;
+ *ptr++ = values[0] >> 9;
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 35;
+ *ptr++ = values[2] >> 27;
+ *ptr++ = values[2] >> 19;
+ *ptr++ = values[2] >> 11;
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 37;
+ *ptr++ = values[4] >> 29;
+ *ptr++ = values[4] >> 21;
+ *ptr++ = values[4] >> 13;
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 39;
+ *ptr++ = values[6] >> 31;
+ *ptr++ = values[6] >> 23;
+ *ptr++ = values[6] >> 15;
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_42(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 34;
+ *ptr++ = values[0] >> 26;
+ *ptr++ = values[0] >> 18;
+ *ptr++ = values[0] >> 10;
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 38;
+ *ptr++ = values[2] >> 30;
+ *ptr++ = values[2] >> 22;
+ *ptr++ = values[2] >> 14;
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 34;
+ *ptr++ = values[4] >> 26;
+ *ptr++ = values[4] >> 18;
+ *ptr++ = values[4] >> 10;
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 38;
+ *ptr++ = values[6] >> 30;
+ *ptr++ = values[6] >> 22;
+ *ptr++ = values[6] >> 14;
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_43(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 35;
+ *ptr++ = values[0] >> 27;
+ *ptr++ = values[0] >> 19;
+ *ptr++ = values[0] >> 11;
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 41;
+ *ptr++ = values[2] >> 33;
+ *ptr++ = values[2] >> 25;
+ *ptr++ = values[2] >> 17;
+ *ptr++ = values[2] >> 9;
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 39;
+ *ptr++ = values[4] >> 31;
+ *ptr++ = values[4] >> 23;
+ *ptr++ = values[4] >> 15;
+ *ptr++ = values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 37;
+ *ptr++ = values[6] >> 29;
+ *ptr++ = values[6] >> 21;
+ *ptr++ = values[6] >> 13;
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_44(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 36;
+ *ptr++ = values[0] >> 28;
+ *ptr++ = values[0] >> 20;
+ *ptr++ = values[0] >> 12;
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 40;
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 36;
+ *ptr++ = values[2] >> 28;
+ *ptr++ = values[2] >> 20;
+ *ptr++ = values[2] >> 12;
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 36;
+ *ptr++ = values[4] >> 28;
+ *ptr++ = values[4] >> 20;
+ *ptr++ = values[4] >> 12;
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 40;
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 36;
+ *ptr++ = values[6] >> 28;
+ *ptr++ = values[6] >> 20;
+ *ptr++ = values[6] >> 12;
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_45(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 37;
+ *ptr++ = values[0] >> 29;
+ *ptr++ = values[0] >> 21;
+ *ptr++ = values[0] >> 13;
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 42;
+ *ptr++ = values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 39;
+ *ptr++ = values[2] >> 31;
+ *ptr++ = values[2] >> 23;
+ *ptr++ = values[2] >> 15;
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 41;
+ *ptr++ = values[4] >> 33;
+ *ptr++ = values[4] >> 25;
+ *ptr++ = values[4] >> 17;
+ *ptr++ = values[4] >> 9;
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 43;
+ *ptr++ = values[6] >> 35;
+ *ptr++ = values[6] >> 27;
+ *ptr++ = values[6] >> 19;
+ *ptr++ = values[6] >> 11;
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_46(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 38;
+ *ptr++ = values[0] >> 30;
+ *ptr++ = values[0] >> 22;
+ *ptr++ = values[0] >> 14;
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 44;
+ *ptr++ = values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 42;
+ *ptr++ = values[2] >> 34;
+ *ptr++ = values[2] >> 26;
+ *ptr++ = values[2] >> 18;
+ *ptr++ = values[2] >> 10;
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 38;
+ *ptr++ = values[4] >> 30;
+ *ptr++ = values[4] >> 22;
+ *ptr++ = values[4] >> 14;
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 44;
+ *ptr++ = values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 42;
+ *ptr++ = values[6] >> 34;
+ *ptr++ = values[6] >> 26;
+ *ptr++ = values[6] >> 18;
+ *ptr++ = values[6] >> 10;
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_47(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 39;
+ *ptr++ = values[0] >> 31;
+ *ptr++ = values[0] >> 23;
+ *ptr++ = values[0] >> 15;
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 46;
+ *ptr++ = values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 45;
+ *ptr++ = values[2] >> 37;
+ *ptr++ = values[2] >> 29;
+ *ptr++ = values[2] >> 21;
+ *ptr++ = values[2] >> 13;
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 43;
+ *ptr++ = values[4] >> 35;
+ *ptr++ = values[4] >> 27;
+ *ptr++ = values[4] >> 19;
+ *ptr++ = values[4] >> 11;
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 41;
+ *ptr++ = values[6] >> 33;
+ *ptr++ = values[6] >> 25;
+ *ptr++ = values[6] >> 17;
+ *ptr++ = values[6] >> 9;
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_48(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 40;
+ *ptr++ = values[0] >> 32;
+ *ptr++ = values[0] >> 24;
+ *ptr++ = values[0] >> 16;
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 40;
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 40;
+ *ptr++ = values[2] >> 32;
+ *ptr++ = values[2] >> 24;
+ *ptr++ = values[2] >> 16;
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 40;
+ *ptr++ = values[4] >> 32;
+ *ptr++ = values[4] >> 24;
+ *ptr++ = values[4] >> 16;
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 40;
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 40;
+ *ptr++ = values[6] >> 32;
+ *ptr++ = values[6] >> 24;
+ *ptr++ = values[6] >> 16;
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_49(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 41;
+ *ptr++ = values[0] >> 33;
+ *ptr++ = values[0] >> 25;
+ *ptr++ = values[0] >> 17;
+ *ptr++ = values[0] >> 9;
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 42;
+ *ptr++ = values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 43;
+ *ptr++ = values[2] >> 35;
+ *ptr++ = values[2] >> 27;
+ *ptr++ = values[2] >> 19;
+ *ptr++ = values[2] >> 11;
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 45;
+ *ptr++ = values[4] >> 37;
+ *ptr++ = values[4] >> 29;
+ *ptr++ = values[4] >> 21;
+ *ptr++ = values[4] >> 13;
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 46;
+ *ptr++ = values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 47;
+ *ptr++ = values[6] >> 39;
+ *ptr++ = values[6] >> 31;
+ *ptr++ = values[6] >> 23;
+ *ptr++ = values[6] >> 15;
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_50(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 42;
+ *ptr++ = values[0] >> 34;
+ *ptr++ = values[0] >> 26;
+ *ptr++ = values[0] >> 18;
+ *ptr++ = values[0] >> 10;
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 44;
+ *ptr++ = values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 46;
+ *ptr++ = values[2] >> 38;
+ *ptr++ = values[2] >> 30;
+ *ptr++ = values[2] >> 22;
+ *ptr++ = values[2] >> 14;
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 42;
+ *ptr++ = values[4] >> 34;
+ *ptr++ = values[4] >> 26;
+ *ptr++ = values[4] >> 18;
+ *ptr++ = values[4] >> 10;
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 44;
+ *ptr++ = values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 46;
+ *ptr++ = values[6] >> 38;
+ *ptr++ = values[6] >> 30;
+ *ptr++ = values[6] >> 22;
+ *ptr++ = values[6] >> 14;
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_51(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 43;
+ *ptr++ = values[0] >> 35;
+ *ptr++ = values[0] >> 27;
+ *ptr++ = values[0] >> 19;
+ *ptr++ = values[0] >> 11;
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 46;
+ *ptr++ = values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 49;
+ *ptr++ = values[2] >> 41;
+ *ptr++ = values[2] >> 33;
+ *ptr++ = values[2] >> 25;
+ *ptr++ = values[2] >> 17;
+ *ptr++ = values[2] >> 9;
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 47;
+ *ptr++ = values[4] >> 39;
+ *ptr++ = values[4] >> 31;
+ *ptr++ = values[4] >> 23;
+ *ptr++ = values[4] >> 15;
+ *ptr++ = values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 50;
+ *ptr++ = values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 45;
+ *ptr++ = values[6] >> 37;
+ *ptr++ = values[6] >> 29;
+ *ptr++ = values[6] >> 21;
+ *ptr++ = values[6] >> 13;
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_52(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 44;
+ *ptr++ = values[0] >> 36;
+ *ptr++ = values[0] >> 28;
+ *ptr++ = values[0] >> 20;
+ *ptr++ = values[0] >> 12;
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 48;
+ *ptr++ = values[1] >> 40;
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 44;
+ *ptr++ = values[2] >> 36;
+ *ptr++ = values[2] >> 28;
+ *ptr++ = values[2] >> 20;
+ *ptr++ = values[2] >> 12;
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 44;
+ *ptr++ = values[4] >> 36;
+ *ptr++ = values[4] >> 28;
+ *ptr++ = values[4] >> 20;
+ *ptr++ = values[4] >> 12;
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 48;
+ *ptr++ = values[5] >> 40;
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 44;
+ *ptr++ = values[6] >> 36;
+ *ptr++ = values[6] >> 28;
+ *ptr++ = values[6] >> 20;
+ *ptr++ = values[6] >> 12;
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_53(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 45;
+ *ptr++ = values[0] >> 37;
+ *ptr++ = values[0] >> 29;
+ *ptr++ = values[0] >> 21;
+ *ptr++ = values[0] >> 13;
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 50;
+ *ptr++ = values[1] >> 42;
+ *ptr++ = values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 47;
+ *ptr++ = values[2] >> 39;
+ *ptr++ = values[2] >> 31;
+ *ptr++ = values[2] >> 23;
+ *ptr++ = values[2] >> 15;
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 49;
+ *ptr++ = values[4] >> 41;
+ *ptr++ = values[4] >> 33;
+ *ptr++ = values[4] >> 25;
+ *ptr++ = values[4] >> 17;
+ *ptr++ = values[4] >> 9;
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 46;
+ *ptr++ = values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 51;
+ *ptr++ = values[6] >> 43;
+ *ptr++ = values[6] >> 35;
+ *ptr++ = values[6] >> 27;
+ *ptr++ = values[6] >> 19;
+ *ptr++ = values[6] >> 11;
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_54(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 46;
+ *ptr++ = values[0] >> 38;
+ *ptr++ = values[0] >> 30;
+ *ptr++ = values[0] >> 22;
+ *ptr++ = values[0] >> 14;
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 52;
+ *ptr++ = values[1] >> 44;
+ *ptr++ = values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 50;
+ *ptr++ = values[2] >> 42;
+ *ptr++ = values[2] >> 34;
+ *ptr++ = values[2] >> 26;
+ *ptr++ = values[2] >> 18;
+ *ptr++ = values[2] >> 10;
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 46;
+ *ptr++ = values[4] >> 38;
+ *ptr++ = values[4] >> 30;
+ *ptr++ = values[4] >> 22;
+ *ptr++ = values[4] >> 14;
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 52;
+ *ptr++ = values[5] >> 44;
+ *ptr++ = values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 50;
+ *ptr++ = values[6] >> 42;
+ *ptr++ = values[6] >> 34;
+ *ptr++ = values[6] >> 26;
+ *ptr++ = values[6] >> 18;
+ *ptr++ = values[6] >> 10;
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_55(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 47;
+ *ptr++ = values[0] >> 39;
+ *ptr++ = values[0] >> 31;
+ *ptr++ = values[0] >> 23;
+ *ptr++ = values[0] >> 15;
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 54;
+ *ptr++ = values[1] >> 46;
+ *ptr++ = values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 53;
+ *ptr++ = values[2] >> 45;
+ *ptr++ = values[2] >> 37;
+ *ptr++ = values[2] >> 29;
+ *ptr++ = values[2] >> 21;
+ *ptr++ = values[2] >> 13;
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 51;
+ *ptr++ = values[4] >> 43;
+ *ptr++ = values[4] >> 35;
+ *ptr++ = values[4] >> 27;
+ *ptr++ = values[4] >> 19;
+ *ptr++ = values[4] >> 11;
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 50;
+ *ptr++ = values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 49;
+ *ptr++ = values[6] >> 41;
+ *ptr++ = values[6] >> 33;
+ *ptr++ = values[6] >> 25;
+ *ptr++ = values[6] >> 17;
+ *ptr++ = values[6] >> 9;
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_56(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 48;
+ *ptr++ = values[0] >> 40;
+ *ptr++ = values[0] >> 32;
+ *ptr++ = values[0] >> 24;
+ *ptr++ = values[0] >> 16;
+ *ptr++ = values[0] >> 8;
+ *ptr++ = values[0];
+
+ *ptr++ = values[1] >> 48;
+ *ptr++ = values[1] >> 40;
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 48;
+ *ptr++ = values[2] >> 40;
+ *ptr++ = values[2] >> 32;
+ *ptr++ = values[2] >> 24;
+ *ptr++ = values[2] >> 16;
+ *ptr++ = values[2] >> 8;
+ *ptr++ = values[2];
+
+ *ptr++ = values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 48;
+ *ptr++ = values[4] >> 40;
+ *ptr++ = values[4] >> 32;
+ *ptr++ = values[4] >> 24;
+ *ptr++ = values[4] >> 16;
+ *ptr++ = values[4] >> 8;
+ *ptr++ = values[4];
+
+ *ptr++ = values[5] >> 48;
+ *ptr++ = values[5] >> 40;
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 48;
+ *ptr++ = values[6] >> 40;
+ *ptr++ = values[6] >> 32;
+ *ptr++ = values[6] >> 24;
+ *ptr++ = values[6] >> 16;
+ *ptr++ = values[6] >> 8;
+ *ptr++ = values[6];
+
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_57(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 49;
+ *ptr++ = values[0] >> 41;
+ *ptr++ = values[0] >> 33;
+ *ptr++ = values[0] >> 25;
+ *ptr++ = values[0] >> 17;
+ *ptr++ = values[0] >> 9;
+ *ptr++ = values[0] >> 1;
+
+ *ptr = values[0] << 7;
+ *ptr++ |= values[1] >> 50;
+ *ptr++ = values[1] >> 42;
+ *ptr++ = values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 51;
+ *ptr++ = values[2] >> 43;
+ *ptr++ = values[2] >> 35;
+ *ptr++ = values[2] >> 27;
+ *ptr++ = values[2] >> 19;
+ *ptr++ = values[2] >> 11;
+ *ptr++ = values[2] >> 3;
+
+ *ptr = values[2] << 5;
+ *ptr++ |= values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 53;
+ *ptr++ = values[4] >> 45;
+ *ptr++ = values[4] >> 37;
+ *ptr++ = values[4] >> 29;
+ *ptr++ = values[4] >> 21;
+ *ptr++ = values[4] >> 13;
+ *ptr++ = values[4] >> 5;
+
+ *ptr = values[4] << 3;
+ *ptr++ |= values[5] >> 54;
+ *ptr++ = values[5] >> 46;
+ *ptr++ = values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 55;
+ *ptr++ = values[6] >> 47;
+ *ptr++ = values[6] >> 39;
+ *ptr++ = values[6] >> 31;
+ *ptr++ = values[6] >> 23;
+ *ptr++ = values[6] >> 15;
+ *ptr++ = values[6] >> 7;
+
+ *ptr = values[6] << 1;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_58(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 50;
+ *ptr++ = values[0] >> 42;
+ *ptr++ = values[0] >> 34;
+ *ptr++ = values[0] >> 26;
+ *ptr++ = values[0] >> 18;
+ *ptr++ = values[0] >> 10;
+ *ptr++ = values[0] >> 2;
+
+ *ptr = values[0] << 6;
+ *ptr++ |= values[1] >> 52;
+ *ptr++ = values[1] >> 44;
+ *ptr++ = values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 54;
+ *ptr++ = values[2] >> 46;
+ *ptr++ = values[2] >> 38;
+ *ptr++ = values[2] >> 30;
+ *ptr++ = values[2] >> 22;
+ *ptr++ = values[2] >> 14;
+ *ptr++ = values[2] >> 6;
+
+ *ptr = values[2] << 2;
+ *ptr++ |= values[3] >> 56;
+ *ptr++ = values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 50;
+ *ptr++ = values[4] >> 42;
+ *ptr++ = values[4] >> 34;
+ *ptr++ = values[4] >> 26;
+ *ptr++ = values[4] >> 18;
+ *ptr++ = values[4] >> 10;
+ *ptr++ = values[4] >> 2;
+
+ *ptr = values[4] << 6;
+ *ptr++ |= values[5] >> 52;
+ *ptr++ = values[5] >> 44;
+ *ptr++ = values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 54;
+ *ptr++ = values[6] >> 46;
+ *ptr++ = values[6] >> 38;
+ *ptr++ = values[6] >> 30;
+ *ptr++ = values[6] >> 22;
+ *ptr++ = values[6] >> 14;
+ *ptr++ = values[6] >> 6;
+
+ *ptr = values[6] << 2;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_59(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 51;
+ *ptr++ = values[0] >> 43;
+ *ptr++ = values[0] >> 35;
+ *ptr++ = values[0] >> 27;
+ *ptr++ = values[0] >> 19;
+ *ptr++ = values[0] >> 11;
+ *ptr++ = values[0] >> 3;
+
+ *ptr = values[0] << 5;
+ *ptr++ |= values[1] >> 54;
+ *ptr++ = values[1] >> 46;
+ *ptr++ = values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 57;
+ *ptr++ = values[2] >> 49;
+ *ptr++ = values[2] >> 41;
+ *ptr++ = values[2] >> 33;
+ *ptr++ = values[2] >> 25;
+ *ptr++ = values[2] >> 17;
+ *ptr++ = values[2] >> 9;
+ *ptr++ = values[2] >> 1;
+
+ *ptr = values[2] << 7;
+ *ptr++ |= values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 55;
+ *ptr++ = values[4] >> 47;
+ *ptr++ = values[4] >> 39;
+ *ptr++ = values[4] >> 31;
+ *ptr++ = values[4] >> 23;
+ *ptr++ = values[4] >> 15;
+ *ptr++ = values[4] >> 7;
+
+ *ptr = values[4] << 1;
+ *ptr++ |= values[5] >> 58;
+ *ptr++ = values[5] >> 50;
+ *ptr++ = values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 53;
+ *ptr++ = values[6] >> 45;
+ *ptr++ = values[6] >> 37;
+ *ptr++ = values[6] >> 29;
+ *ptr++ = values[6] >> 21;
+ *ptr++ = values[6] >> 13;
+ *ptr++ = values[6] >> 5;
+
+ *ptr = values[6] << 3;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_60(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 52;
+ *ptr++ = values[0] >> 44;
+ *ptr++ = values[0] >> 36;
+ *ptr++ = values[0] >> 28;
+ *ptr++ = values[0] >> 20;
+ *ptr++ = values[0] >> 12;
+ *ptr++ = values[0] >> 4;
+
+ *ptr = values[0] << 4;
+ *ptr++ |= values[1] >> 56;
+ *ptr++ = values[1] >> 48;
+ *ptr++ = values[1] >> 40;
+ *ptr++ = values[1] >> 32;
+ *ptr++ = values[1] >> 24;
+ *ptr++ = values[1] >> 16;
+ *ptr++ = values[1] >> 8;
+ *ptr++ = values[1];
+
+ *ptr++ = values[2] >> 52;
+ *ptr++ = values[2] >> 44;
+ *ptr++ = values[2] >> 36;
+ *ptr++ = values[2] >> 28;
+ *ptr++ = values[2] >> 20;
+ *ptr++ = values[2] >> 12;
+ *ptr++ = values[2] >> 4;
+
+ *ptr = values[2] << 4;
+ *ptr++ |= values[3] >> 56;
+ *ptr++ = values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 52;
+ *ptr++ = values[4] >> 44;
+ *ptr++ = values[4] >> 36;
+ *ptr++ = values[4] >> 28;
+ *ptr++ = values[4] >> 20;
+ *ptr++ = values[4] >> 12;
+ *ptr++ = values[4] >> 4;
+
+ *ptr = values[4] << 4;
+ *ptr++ |= values[5] >> 56;
+ *ptr++ = values[5] >> 48;
+ *ptr++ = values[5] >> 40;
+ *ptr++ = values[5] >> 32;
+ *ptr++ = values[5] >> 24;
+ *ptr++ = values[5] >> 16;
+ *ptr++ = values[5] >> 8;
+ *ptr++ = values[5];
+
+ *ptr++ = values[6] >> 52;
+ *ptr++ = values[6] >> 44;
+ *ptr++ = values[6] >> 36;
+ *ptr++ = values[6] >> 28;
+ *ptr++ = values[6] >> 20;
+ *ptr++ = values[6] >> 12;
+ *ptr++ = values[6] >> 4;
+
+ *ptr = values[6] << 4;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_61(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 53;
+ *ptr++ = values[0] >> 45;
+ *ptr++ = values[0] >> 37;
+ *ptr++ = values[0] >> 29;
+ *ptr++ = values[0] >> 21;
+ *ptr++ = values[0] >> 13;
+ *ptr++ = values[0] >> 5;
+
+ *ptr = values[0] << 3;
+ *ptr++ |= values[1] >> 58;
+ *ptr++ = values[1] >> 50;
+ *ptr++ = values[1] >> 42;
+ *ptr++ = values[1] >> 34;
+ *ptr++ = values[1] >> 26;
+ *ptr++ = values[1] >> 18;
+ *ptr++ = values[1] >> 10;
+ *ptr++ = values[1] >> 2;
+
+ *ptr = values[1] << 6;
+ *ptr++ |= values[2] >> 55;
+ *ptr++ = values[2] >> 47;
+ *ptr++ = values[2] >> 39;
+ *ptr++ = values[2] >> 31;
+ *ptr++ = values[2] >> 23;
+ *ptr++ = values[2] >> 15;
+ *ptr++ = values[2] >> 7;
+
+ *ptr = values[2] << 1;
+ *ptr++ |= values[3] >> 60;
+ *ptr++ = values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 57;
+ *ptr++ = values[4] >> 49;
+ *ptr++ = values[4] >> 41;
+ *ptr++ = values[4] >> 33;
+ *ptr++ = values[4] >> 25;
+ *ptr++ = values[4] >> 17;
+ *ptr++ = values[4] >> 9;
+ *ptr++ = values[4] >> 1;
+
+ *ptr = values[4] << 7;
+ *ptr++ |= values[5] >> 54;
+ *ptr++ = values[5] >> 46;
+ *ptr++ = values[5] >> 38;
+ *ptr++ = values[5] >> 30;
+ *ptr++ = values[5] >> 22;
+ *ptr++ = values[5] >> 14;
+ *ptr++ = values[5] >> 6;
+
+ *ptr = values[5] << 2;
+ *ptr++ |= values[6] >> 59;
+ *ptr++ = values[6] >> 51;
+ *ptr++ = values[6] >> 43;
+ *ptr++ = values[6] >> 35;
+ *ptr++ = values[6] >> 27;
+ *ptr++ = values[6] >> 19;
+ *ptr++ = values[6] >> 11;
+ *ptr++ = values[6] >> 3;
+
+ *ptr = values[6] << 5;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_62(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 54;
+ *ptr++ = values[0] >> 46;
+ *ptr++ = values[0] >> 38;
+ *ptr++ = values[0] >> 30;
+ *ptr++ = values[0] >> 22;
+ *ptr++ = values[0] >> 14;
+ *ptr++ = values[0] >> 6;
+
+ *ptr = values[0] << 2;
+ *ptr++ |= values[1] >> 60;
+ *ptr++ = values[1] >> 52;
+ *ptr++ = values[1] >> 44;
+ *ptr++ = values[1] >> 36;
+ *ptr++ = values[1] >> 28;
+ *ptr++ = values[1] >> 20;
+ *ptr++ = values[1] >> 12;
+ *ptr++ = values[1] >> 4;
+
+ *ptr = values[1] << 4;
+ *ptr++ |= values[2] >> 58;
+ *ptr++ = values[2] >> 50;
+ *ptr++ = values[2] >> 42;
+ *ptr++ = values[2] >> 34;
+ *ptr++ = values[2] >> 26;
+ *ptr++ = values[2] >> 18;
+ *ptr++ = values[2] >> 10;
+ *ptr++ = values[2] >> 2;
+
+ *ptr = values[2] << 6;
+ *ptr++ |= values[3] >> 56;
+ *ptr++ = values[3] >> 48;
+ *ptr++ = values[3] >> 40;
+ *ptr++ = values[3] >> 32;
+ *ptr++ = values[3] >> 24;
+ *ptr++ = values[3] >> 16;
+ *ptr++ = values[3] >> 8;
+ *ptr++ = values[3];
+
+ *ptr++ = values[4] >> 54;
+ *ptr++ = values[4] >> 46;
+ *ptr++ = values[4] >> 38;
+ *ptr++ = values[4] >> 30;
+ *ptr++ = values[4] >> 22;
+ *ptr++ = values[4] >> 14;
+ *ptr++ = values[4] >> 6;
+
+ *ptr = values[4] << 2;
+ *ptr++ |= values[5] >> 60;
+ *ptr++ = values[5] >> 52;
+ *ptr++ = values[5] >> 44;
+ *ptr++ = values[5] >> 36;
+ *ptr++ = values[5] >> 28;
+ *ptr++ = values[5] >> 20;
+ *ptr++ = values[5] >> 12;
+ *ptr++ = values[5] >> 4;
+
+ *ptr = values[5] << 4;
+ *ptr++ |= values[6] >> 58;
+ *ptr++ = values[6] >> 50;
+ *ptr++ = values[6] >> 42;
+ *ptr++ = values[6] >> 34;
+ *ptr++ = values[6] >> 26;
+ *ptr++ = values[6] >> 18;
+ *ptr++ = values[6] >> 10;
+ *ptr++ = values[6] >> 2;
+
+ *ptr = values[6] << 6;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void pack_bits_63(const uint64_t* values, uint8_t* ptr) {
+ *ptr++ = values[0] >> 55;
+ *ptr++ = values[0] >> 47;
+ *ptr++ = values[0] >> 39;
+ *ptr++ = values[0] >> 31;
+ *ptr++ = values[0] >> 23;
+ *ptr++ = values[0] >> 15;
+ *ptr++ = values[0] >> 7;
+
+ *ptr = values[0] << 1;
+ *ptr++ |= values[1] >> 62;
+ *ptr++ = values[1] >> 54;
+ *ptr++ = values[1] >> 46;
+ *ptr++ = values[1] >> 38;
+ *ptr++ = values[1] >> 30;
+ *ptr++ = values[1] >> 22;
+ *ptr++ = values[1] >> 14;
+ *ptr++ = values[1] >> 6;
+
+ *ptr = values[1] << 2;
+ *ptr++ |= values[2] >> 61;
+ *ptr++ = values[2] >> 53;
+ *ptr++ = values[2] >> 45;
+ *ptr++ = values[2] >> 37;
+ *ptr++ = values[2] >> 29;
+ *ptr++ = values[2] >> 21;
+ *ptr++ = values[2] >> 13;
+ *ptr++ = values[2] >> 5;
+
+ *ptr = values[2] << 3;
+ *ptr++ |= values[3] >> 60;
+ *ptr++ = values[3] >> 52;
+ *ptr++ = values[3] >> 44;
+ *ptr++ = values[3] >> 36;
+ *ptr++ = values[3] >> 28;
+ *ptr++ = values[3] >> 20;
+ *ptr++ = values[3] >> 12;
+ *ptr++ = values[3] >> 4;
+
+ *ptr = values[3] << 4;
+ *ptr++ |= values[4] >> 59;
+ *ptr++ = values[4] >> 51;
+ *ptr++ = values[4] >> 43;
+ *ptr++ = values[4] >> 35;
+ *ptr++ = values[4] >> 27;
+ *ptr++ = values[4] >> 19;
+ *ptr++ = values[4] >> 11;
+ *ptr++ = values[4] >> 3;
+
+ *ptr = values[4] << 5;
+ *ptr++ |= values[5] >> 58;
+ *ptr++ = values[5] >> 50;
+ *ptr++ = values[5] >> 42;
+ *ptr++ = values[5] >> 34;
+ *ptr++ = values[5] >> 26;
+ *ptr++ = values[5] >> 18;
+ *ptr++ = values[5] >> 10;
+ *ptr++ = values[5] >> 2;
+
+ *ptr = values[5] << 6;
+ *ptr++ |= values[6] >> 57;
+ *ptr++ = values[6] >> 49;
+ *ptr++ = values[6] >> 41;
+ *ptr++ = values[6] >> 33;
+ *ptr++ = values[6] >> 25;
+ *ptr++ = values[6] >> 17;
+ *ptr++ = values[6] >> 9;
+ *ptr++ = values[6] >> 1;
+
+ *ptr = values[6] << 7;
+ *ptr++ |= values[7] >> 56;
+ *ptr++ = values[7] >> 48;
+ *ptr++ = values[7] >> 40;
+ *ptr++ = values[7] >> 32;
+ *ptr++ = values[7] >> 24;
+ *ptr++ = values[7] >> 16;
+ *ptr++ = values[7] >> 8;
+ *ptr = values[7];
+}
+
+static inline void unpack_bits_1(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 7;
+ values[1] = (*ptr >> 6) & 1;
+ values[2] = (*ptr >> 5) & 1;
+ values[3] = (*ptr >> 4) & 1;
+ values[4] = (*ptr >> 3) & 1;
+ values[5] = (*ptr >> 2) & 1;
+ values[6] = (*ptr >> 1) & 1;
+ values[7] = *ptr++ & 1;
+}
+
+static inline void unpack_bits_2(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 6;
+ values[1] = (*ptr >> 4) & 3;
+ values[2] = (*ptr >> 2) & 3;
+ values[3] = *ptr++ & 3;
+ values[4] = *ptr >> 6;
+ values[5] = (*ptr >> 4) & 3;
+ values[6] = (*ptr >> 2) & 3;
+ values[7] = *ptr & 3;
+}
+
+static inline void unpack_bits_3(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 5;
+ values[1] = (*ptr >> 2) & 7;
+ values[2] = (*ptr++ & 3) << 1;
+ values[2] |= *ptr >> 7;
+ values[3] = (*ptr >> 4) & 7;
+ values[4] = (*ptr >> 1) & 7;
+ values[5] = (*ptr++ & 1) << 2;
+ values[5] |= *ptr >> 6;
+ values[6] = (*ptr >> 3) & 7;
+ values[7] = *ptr & 7;
+}
+
+static inline void unpack_bits_4(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 4;
+ values[1] = *ptr++ & 0xf;
+ values[2] = *ptr >> 4;
+ values[3] = *ptr++ & 0xf;
+ values[4] = *ptr >> 4;
+ values[5] = *ptr++ & 0xf;
+ values[6] = *ptr >> 4;
+ values[7] = *ptr & 0xf;
+}
+
+static inline void unpack_bits_5(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 3;
+
+ values[1] = (*ptr++ & 7) << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr >> 1) & 0x1f;
+
+ values[3] = (*ptr++ & 1) << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = (*ptr >> 2) & 0x1f;
+
+ values[6] = (*ptr++ & 3) << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = *ptr & 0x1f;
+}
+
+static inline void unpack_bits_6(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 2;
+
+ values[1] = (*ptr++ & 3) << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = *ptr++ & 0x3f;
+
+ values[4] = *ptr >> 2;
+
+ values[5] = (*ptr++ & 3) << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = *ptr & 0x3f;
+}
+
+static inline void unpack_bits_7(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr >> 1;
+
+ values[1] = (*ptr++ & 1) << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = (*ptr++ & 3) << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = (*ptr++ & 7) << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = (*ptr++ & 0x1f) << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = *ptr & 0x7f;
+}
+
+static inline void unpack_bits_8(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++;
+ values[1] = *ptr++;
+ values[2] = *ptr++;
+ values[3] = *ptr++;
+ values[4] = *ptr++;
+ values[5] = *ptr++;
+ values[6] = *ptr++;
+ values[7] = *ptr;
+}
+
+static inline void unpack_bits_9(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = (*ptr++ & 0x7f) << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = (*ptr++ & 0x1f) << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = (*ptr++ & 7) << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = (*ptr++ & 3) << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = (*ptr++ & 1) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_10(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = (*ptr++ & 0x3f) << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = (*ptr++ & 3) << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = (*ptr++ & 0x3f) << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = (*ptr++ & 3) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_11(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = (*ptr++ & 0x1f) << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = (*ptr++ & 3) << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = (*ptr++ & 0x7f) << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = (*ptr++ & 1) << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = (*ptr++ & 7) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_12(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = (*ptr++ & 0xf) << 8;
+ values[1] |= *ptr++;
+
+ values[2] = *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = (*ptr++ & 0xf) << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = (*ptr++ & 0xf) << 8;
+ values[5] |= *ptr++;
+
+ values[6] = *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = (*ptr++ & 0xf) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_13(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = (*ptr++ & 7) << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = (*ptr++ & 1) << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = (*ptr++ & 0x7f) << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = (*ptr++ & 3) << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = (*ptr++ & 0x1f) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_14(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = (*ptr++ & 3) << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = (*ptr++ & 0x3f) << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = (*ptr++ & 3) << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = (*ptr++ & 0x3f) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_15(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = (*ptr++ & 1) << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = (*ptr++ & 3) << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = (*ptr++ & 7) << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = (*ptr++ & 0x1f) << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = (*ptr++ & 0x7f) << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_16(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_17(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = (*ptr++ & 0x7f) << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = (*ptr++ & 0x1f) << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = (*ptr++ & 7) << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = (*ptr++ & 3) << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = (*ptr++ & 1) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_18(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = (*ptr++ & 0x3f) << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = (*ptr++ & 3) << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = (*ptr++ & 0x3f) << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = (*ptr++ & 3) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_19(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = (*ptr++ & 0x1f) << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = (*ptr++ & 3) << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = (*ptr++ & 0x7f) << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = (*ptr++ & 1) << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = (*ptr++ & 7) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_20(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = (*ptr++ & 0xf) << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = (*ptr++ & 0xf) << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = (*ptr++ & 0xf) << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = (*ptr++ & 0xf) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_21(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = (*ptr++ & 7) << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = (*ptr++ & 1) << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = (*ptr++ & 0x7f) << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = (*ptr++ & 3) << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = (*ptr++ & 0x1f) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_22(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = (*ptr++ & 3) << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = (*ptr++ & 0x3f) << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = (*ptr++ & 3) << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = (*ptr++ & 0x3f) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_23(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = (*ptr++ & 1) << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = (*ptr++ & 3) << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = (*ptr++ & 7) << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = (*ptr++ & 0x1f) << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = (*ptr++ & 0x7f) << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_24(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 16;
+ values[0] |= *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = *ptr++ << 16;
+ values[2] |= *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = *ptr++ << 16;
+ values[4] |= *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = *ptr++ << 16;
+ values[6] |= *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_25(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 17;
+ values[0] |= *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = (*ptr++ & 0x7f) << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 19;
+ values[2] |= *ptr++ << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = (*ptr++ & 0x1f) << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 21;
+ values[4] |= *ptr++ << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = (*ptr++ & 7) << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = (*ptr++ & 3) << 23;
+ values[6] |= *ptr++ << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 1) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_26(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 18;
+ values[0] |= *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = (*ptr++ & 0x3f) << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = (*ptr++ & 0xf) << 22;
+ values[2] |= *ptr++ << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 3) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 18;
+ values[4] |= *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = (*ptr++ & 0x3f) << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = (*ptr++ & 0xf) << 22;
+ values[6] |= *ptr++ << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 3) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_27(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 19;
+ values[0] |= *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = (*ptr++ & 0x1f) << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 25;
+ values[2] |= *ptr++ << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = (*ptr++ & 0x7f) << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = (*ptr++ & 0xf) << 23;
+ values[4] |= *ptr++ << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 1) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = (*ptr++ & 0x3f) << 21;
+ values[6] |= *ptr++ << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 7) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_28(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 20;
+ values[0] |= *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0xf) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = *ptr++ << 20;
+ values[2] |= *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0xf) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 20;
+ values[4] |= *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0xf) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = *ptr++ << 20;
+ values[6] |= *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0xf) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_29(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 21;
+ values[0] |= *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 7) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = (*ptr++ & 0x3f) << 23;
+ values[2] |= *ptr++ << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 1) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 25;
+ values[4] |= *ptr++ << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = (*ptr++ & 0x7f) << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 27;
+ values[6] |= *ptr++ << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x1f) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_30(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 22;
+ values[0] |= *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 3) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 26;
+ values[2] |= *ptr++ << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x3f) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = *ptr++ << 22;
+ values[4] |= *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 3) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 26;
+ values[6] |= *ptr++ << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x3f) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_31(uint64_t* values, const uint8_t* ptr) {
+ values[0] = *ptr++ << 23;
+ values[0] |= *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 1) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 29;
+ values[2] |= *ptr++ << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 7) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 27;
+ values[4] |= *ptr++ << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x1f) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 25;
+ values[6] |= *ptr++ << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x7f) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_32(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 24;
+ values[0] |= *ptr++ << 16;
+ values[0] |= *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = static_cast<uint64_t>(*ptr++) << 24;
+ values[2] |= *ptr++ << 16;
+ values[2] |= *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = static_cast<uint64_t>(*ptr++) << 24;
+ values[4] |= *ptr++ << 16;
+ values[4] |= *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = static_cast<uint64_t>(*ptr++) << 24;
+ values[6] |= *ptr++ << 16;
+ values[6] |= *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_33(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 25;
+ values[0] |= *ptr++ << 17;
+ values[0] |= *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x7f) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 27;
+ values[2] |= *ptr++ << 19;
+ values[2] |= *ptr++ << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x1f) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 29;
+ values[4] |= *ptr++ << 21;
+ values[4] |= *ptr++ << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 7) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 31;
+ values[6] |= *ptr++ << 23;
+ values[6] |= *ptr++ << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 1) << 32;
+ values[7] |= *ptr++ << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_34(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 26;
+ values[0] |= *ptr++ << 18;
+ values[0] |= *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x3f) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 30;
+ values[2] |= *ptr++ << 22;
+ values[2] |= *ptr++ << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 3) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 26;
+ values[4] |= *ptr++ << 18;
+ values[4] |= *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x3f) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 30;
+ values[6] |= *ptr++ << 22;
+ values[6] |= *ptr++ << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 3) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr++;
+}
+
+static inline void unpack_bits_35(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 27;
+ values[0] |= *ptr++ << 19;
+ values[0] |= *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x1f) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 2) << 33;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[2] |= *ptr++ << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x7f) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 31;
+ values[4] |= *ptr++ << 23;
+ values[4] |= *ptr++ << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 1) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 29;
+ values[6] |= *ptr++ << 21;
+ values[6] |= *ptr++ << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 7) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_36(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 28;
+ values[0] |= *ptr++ << 20;
+ values[0] |= *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0xf) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = static_cast<uint64_t>(*ptr++) << 28;
+ values[2] |= *ptr++ << 20;
+ values[2] |= *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0xf) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 28;
+ values[4] |= *ptr++ << 20;
+ values[4] |= *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0xf) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = static_cast<uint64_t>(*ptr++) << 28;
+ values[6] |= *ptr++ << 20;
+ values[6] |= *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0xf) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_37(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 29;
+ values[0] |= *ptr++ << 21;
+ values[0] |= *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 7) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 31;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 23;
+ values[2] |= *ptr++ << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 1) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 33;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[4] |= *ptr++ << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x7f) << 30;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 35;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[6] |= *ptr++ << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x1f) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_38(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 30;
+ values[0] |= *ptr++ << 22;
+ values[0] |= *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 3) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 34;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[2] |= *ptr++ << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x3f) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 30;
+ values[4] |= *ptr++ << 22;
+ values[4] |= *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 3) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 34;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[6] |= *ptr++ << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x3f) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_39(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 31;
+ values[0] |= *ptr++ << 23;
+ values[0] |= *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 1) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 37;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[2] |= *ptr++ << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 7) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 35;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[4] |= *ptr++ << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x1f) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 33;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[6] |= *ptr++ << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x7f) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_40(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 32;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[0] |= *ptr++ << 16;
+ values[0] |= *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = static_cast<uint64_t>(*ptr++) << 32;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[2] |= *ptr++ << 16;
+ values[2] |= *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = static_cast<uint64_t>(*ptr++) << 32;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[4] |= *ptr++ << 16;
+ values[4] |= *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = static_cast<uint64_t>(*ptr++) << 32;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[6] |= *ptr++ << 16;
+ values[6] |= *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_41(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 33;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[0] |= *ptr++ << 17;
+ values[0] |= *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x7f) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 35;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[2] |= *ptr++ << 19;
+ values[2] |= *ptr++ << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x1f) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 37;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[4] |= *ptr++ << 21;
+ values[4] |= *ptr++ << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 7) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 39;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[6] |= *ptr++ << 23;
+ values[6] |= *ptr++ << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 1) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_42(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 34;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[0] |= *ptr++ << 18;
+ values[0] |= *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x3f) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 38;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[2] |= *ptr++ << 22;
+ values[2] |= *ptr++ << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 3) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 34;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[4] |= *ptr++ << 18;
+ values[4] |= *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x3f) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 38;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[6] |= *ptr++ << 22;
+ values[6] |= *ptr++ << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 3) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_43(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 35;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[0] |= *ptr++ << 19;
+ values[0] |= *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x1f) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 41;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[2] |= *ptr++ << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x7f) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 39;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[4] |= *ptr++ << 23;
+ values[4] |= *ptr++ << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 1) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 37;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[6] |= *ptr++ << 21;
+ values[6] |= *ptr++ << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 7) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_44(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 36;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[0] |= *ptr++ << 20;
+ values[0] |= *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0xf) << 40;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = static_cast<uint64_t>(*ptr++) << 36;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[2] |= *ptr++ << 20;
+ values[2] |= *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0xf) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 36;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[4] |= *ptr++ << 20;
+ values[4] |= *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0xf) << 40;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = static_cast<uint64_t>(*ptr++) << 36;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[6] |= *ptr++ << 20;
+ values[6] |= *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0xf) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_45(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 37;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[0] |= *ptr++ << 21;
+ values[0] |= *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 7) << 42;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 39;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 23;
+ values[2] |= *ptr++ << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 1) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 41;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[4] |= *ptr++ << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x7f) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 43;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[6] |= *ptr++ << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x1f) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_46(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 38;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[0] |= *ptr++ << 22;
+ values[0] |= *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 3) << 44;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 42;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[2] |= *ptr++ << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x3f) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 38;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[4] |= *ptr++ << 22;
+ values[4] |= *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 3) << 44;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 42;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[6] |= *ptr++ << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x3f) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_47(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 39;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[0] |= *ptr++ << 23;
+ values[0] |= *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 1) << 46;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 45;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[2] |= *ptr++ << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 7) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 43;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[4] |= *ptr++ << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x1f) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 41;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[6] |= *ptr++ << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x7f) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_48(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 40;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[0] |= *ptr++ << 16;
+ values[0] |= *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = static_cast<uint64_t>(*ptr++) << 40;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = static_cast<uint64_t>(*ptr++) << 40;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[2] |= *ptr++ << 16;
+ values[2] |= *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = static_cast<uint64_t>(*ptr++) << 40;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[4] |= *ptr++ << 16;
+ values[4] |= *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = static_cast<uint64_t>(*ptr++) << 40;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = static_cast<uint64_t>(*ptr++) << 40;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[6] |= *ptr++ << 16;
+ values[6] |= *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_49(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 41;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[0] |= *ptr++ << 17;
+ values[0] |= *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x7f) << 42;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 43;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[2] |= *ptr++ << 19;
+ values[2] |= *ptr++ << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x1f) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 45;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[4] |= *ptr++ << 21;
+ values[4] |= *ptr++ << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 7) << 46;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 47;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[6] |= *ptr++ << 23;
+ values[6] |= *ptr++ << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 1) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_50(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 42;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[0] |= *ptr++ << 18;
+ values[0] |= *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x3f) << 44;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 46;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[2] |= *ptr++ << 22;
+ values[2] |= *ptr++ << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 3) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 42;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[4] |= *ptr++ << 18;
+ values[4] |= *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x3f) << 44;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 46;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[6] |= *ptr++ << 22;
+ values[6] |= *ptr++ << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 3) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_51(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 43;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[0] |= *ptr++ << 19;
+ values[0] |= *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x1f) << 46;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 49;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[2] |= *ptr++ << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x7f) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 47;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[4] |= *ptr++ << 23;
+ values[4] |= *ptr++ << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 1) << 50;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 45;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[6] |= *ptr++ << 21;
+ values[6] |= *ptr++ << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 7) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_52(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 44;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[0] |= *ptr++ << 20;
+ values[0] |= *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0xf) << 48;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = static_cast<uint64_t>(*ptr++) << 44;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[2] |= *ptr++ << 20;
+ values[2] |= *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0xf) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 44;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[4] |= *ptr++ << 20;
+ values[4] |= *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0xf) << 48;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = static_cast<uint64_t>(*ptr++) << 44;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[6] |= *ptr++ << 20;
+ values[6] |= *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0xf) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_53(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 45;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[0] |= *ptr++ << 21;
+ values[0] |= *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 7) << 50;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 47;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 23;
+ values[2] |= *ptr++ << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 1) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 49;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[4] |= *ptr++ << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x7f) << 46;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 51;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[6] |= *ptr++ << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x1f) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_54(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 46;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[0] |= *ptr++ << 22;
+ values[0] |= *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 3) << 52;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 50;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[2] |= *ptr++ << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x3f) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 46;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[4] |= *ptr++ << 22;
+ values[4] |= *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 3) << 52;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 50;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[6] |= *ptr++ << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x3f) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr++;
+}
+
+static inline void unpack_bits_55(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 47;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[0] |= *ptr++ << 23;
+ values[0] |= *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 1) << 54;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 53;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 45;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[2] |= *ptr++ << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 7) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 51;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[4] |= *ptr++ << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x1f) << 50;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 49;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[6] |= *ptr++ << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x7f) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_56(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 48;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[0] |= *ptr++ << 16;
+ values[0] |= *ptr++ << 8;
+ values[0] |= *ptr++;
+ values[1] = static_cast<uint64_t>(*ptr++) << 48;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+ values[2] = static_cast<uint64_t>(*ptr++) << 48;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[2] |= *ptr++ << 16;
+ values[2] |= *ptr++ << 8;
+ values[2] |= *ptr++;
+ values[3] = static_cast<uint64_t>(*ptr++) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+ values[4] = static_cast<uint64_t>(*ptr++) << 48;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[4] |= *ptr++ << 16;
+ values[4] |= *ptr++ << 8;
+ values[4] |= *ptr++;
+ values[5] = static_cast<uint64_t>(*ptr++) << 48;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+ values[6] = static_cast<uint64_t>(*ptr++) << 48;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[6] |= *ptr++ << 16;
+ values[6] |= *ptr++ << 8;
+ values[6] |= *ptr++;
+ values[7] = static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_57(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 49;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[0] |= *ptr++ << 17;
+ values[0] |= *ptr++ << 9;
+ values[0] |= *ptr++ << 1;
+ values[0] |= *ptr >> 7;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x7f) << 50;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 51;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[2] |= *ptr++ << 19;
+ values[2] |= *ptr++ << 11;
+ values[2] |= *ptr++ << 3;
+ values[2] |= *ptr >> 5;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x1f) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 53;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 45;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[4] |= *ptr++ << 21;
+ values[4] |= *ptr++ << 13;
+ values[4] |= *ptr++ << 5;
+ values[4] |= *ptr >> 3;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 7) << 54;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 55;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 47;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[6] |= *ptr++ << 23;
+ values[6] |= *ptr++ << 15;
+ values[6] |= *ptr++ << 7;
+ values[6] |= *ptr >> 1;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 1) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_58(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 50;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[0] |= *ptr++ << 18;
+ values[0] |= *ptr++ << 10;
+ values[0] |= *ptr++ << 2;
+ values[0] |= *ptr >> 6;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x3f) << 52;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 54;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[2] |= *ptr++ << 22;
+ values[2] |= *ptr++ << 14;
+ values[2] |= *ptr++ << 6;
+ values[2] |= *ptr >> 2;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 3) << 56;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 50;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[4] |= *ptr++ << 18;
+ values[4] |= *ptr++ << 10;
+ values[4] |= *ptr++ << 2;
+ values[4] |= *ptr >> 6;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x3f) << 52;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 54;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[6] |= *ptr++ << 22;
+ values[6] |= *ptr++ << 14;
+ values[6] |= *ptr++ << 6;
+ values[6] |= *ptr >> 2;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 3) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr++;
+}
+
+static inline void unpack_bits_59(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 51;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[0] |= *ptr++ << 19;
+ values[0] |= *ptr++ << 11;
+ values[0] |= *ptr++ << 3;
+ values[0] |= *ptr >> 5;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0x1f) << 54;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 57;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 49;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[2] |= *ptr++ << 17;
+ values[2] |= *ptr++ << 9;
+ values[2] |= *ptr++ << 1;
+ values[2] |= *ptr >> 7;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x7f) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 55;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 47;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[4] |= *ptr++ << 23;
+ values[4] |= *ptr++ << 15;
+ values[4] |= *ptr++ << 7;
+ values[4] |= *ptr >> 1;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 1) << 58;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 50;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 53;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 45;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[6] |= *ptr++ << 21;
+ values[6] |= *ptr++ << 13;
+ values[6] |= *ptr++ << 5;
+ values[6] |= *ptr >> 3;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 7) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_60(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 52;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[0] |= *ptr++ << 20;
+ values[0] |= *ptr++ << 12;
+ values[0] |= *ptr++ << 4;
+ values[0] |= *ptr >> 4;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 0xf) << 56;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[1] |= *ptr++ << 16;
+ values[1] |= *ptr++ << 8;
+ values[1] |= *ptr++;
+
+ values[2] = static_cast<uint64_t>(*ptr++) << 52;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[2] |= *ptr++ << 20;
+ values[2] |= *ptr++ << 12;
+ values[2] |= *ptr++ << 4;
+ values[2] |= *ptr >> 4;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0xf) << 56;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 52;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[4] |= *ptr++ << 20;
+ values[4] |= *ptr++ << 12;
+ values[4] |= *ptr++ << 4;
+ values[4] |= *ptr >> 4;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0xf) << 56;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[5] |= *ptr++ << 16;
+ values[5] |= *ptr++ << 8;
+ values[5] |= *ptr++;
+
+ values[6] = static_cast<uint64_t>(*ptr++) << 52;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[6] |= *ptr++ << 20;
+ values[6] |= *ptr++ << 12;
+ values[6] |= *ptr++ << 4;
+ values[6] |= *ptr >> 4;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0xf) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_61(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 53;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 45;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[0] |= *ptr++ << 21;
+ values[0] |= *ptr++ << 13;
+ values[0] |= *ptr++ << 5;
+ values[0] |= *ptr >> 3;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 7) << 58;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 50;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[1] |= *ptr++ << 18;
+ values[1] |= *ptr++ << 10;
+ values[1] |= *ptr++ << 2;
+ values[1] |= *ptr >> 6;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0x3f) << 55;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 47;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[2] |= *ptr++ << 23;
+ values[2] |= *ptr++ << 15;
+ values[2] |= *ptr++ << 7;
+ values[2] |= *ptr >> 1;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 1) << 60;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 57;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 49;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[4] |= *ptr++ << 17;
+ values[4] |= *ptr++ << 9;
+ values[4] |= *ptr++ << 1;
+ values[4] |= *ptr >> 7;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x7f) << 54;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[5] |= *ptr++ << 22;
+ values[5] |= *ptr++ << 14;
+ values[5] |= *ptr++ << 6;
+ values[5] |= *ptr >> 2;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 3) << 59;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 51;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[6] |= *ptr++ << 19;
+ values[6] |= *ptr++ << 11;
+ values[6] |= *ptr++ << 3;
+ values[6] |= *ptr >> 5;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x1f) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_62(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 54;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[0] |= *ptr++ << 22;
+ values[0] |= *ptr++ << 14;
+ values[0] |= *ptr++ << 6;
+ values[0] |= *ptr >> 2;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 3) << 60;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 52;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[1] |= *ptr++ << 20;
+ values[1] |= *ptr++ << 12;
+ values[1] |= *ptr++ << 4;
+ values[1] |= *ptr >> 4;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 0xf) << 58;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 50;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[2] |= *ptr++ << 18;
+ values[2] |= *ptr++ << 10;
+ values[2] |= *ptr++ << 2;
+ values[2] |= *ptr >> 6;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 0x3f) << 56;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[3] |= *ptr++ << 16;
+ values[3] |= *ptr++ << 8;
+ values[3] |= *ptr++;
+
+ values[4] = static_cast<uint64_t>(*ptr++) << 54;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[4] |= *ptr++ << 22;
+ values[4] |= *ptr++ << 14;
+ values[4] |= *ptr++ << 6;
+ values[4] |= *ptr >> 2;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 3) << 60;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 52;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[5] |= *ptr++ << 20;
+ values[5] |= *ptr++ << 12;
+ values[5] |= *ptr++ << 4;
+ values[5] |= *ptr >> 4;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0xf) << 58;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 50;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[6] |= *ptr++ << 18;
+ values[6] |= *ptr++ << 10;
+ values[6] |= *ptr++ << 2;
+ values[6] |= *ptr >> 6;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x3f) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void unpack_bits_63(uint64_t* values, const uint8_t* ptr) {
+ values[0] = static_cast<uint64_t>(*ptr++) << 55;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 47;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 39;
+ values[0] |= static_cast<uint64_t>(*ptr++) << 31;
+ values[0] |= *ptr++ << 23;
+ values[0] |= *ptr++ << 15;
+ values[0] |= *ptr++ << 7;
+ values[0] |= *ptr >> 1;
+
+ values[1] = static_cast<uint64_t>(*ptr++ & 1) << 62;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 54;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 46;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 38;
+ values[1] |= static_cast<uint64_t>(*ptr++) << 30;
+ values[1] |= *ptr++ << 22;
+ values[1] |= *ptr++ << 14;
+ values[1] |= *ptr++ << 6;
+ values[1] |= *ptr >> 2;
+
+ values[2] = static_cast<uint64_t>(*ptr++ & 3) << 61;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 53;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 45;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 37;
+ values[2] |= static_cast<uint64_t>(*ptr++) << 29;
+ values[2] |= *ptr++ << 21;
+ values[2] |= *ptr++ << 13;
+ values[2] |= *ptr++ << 5;
+ values[2] |= *ptr >> 3;
+
+ values[3] = static_cast<uint64_t>(*ptr++ & 7) << 60;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 52;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 44;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 36;
+ values[3] |= static_cast<uint64_t>(*ptr++) << 28;
+ values[3] |= *ptr++ << 20;
+ values[3] |= *ptr++ << 12;
+ values[3] |= *ptr++ << 4;
+ values[3] |= *ptr >> 4;
+
+ values[4] = static_cast<uint64_t>(*ptr++ & 0xf) << 59;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 51;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 43;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 35;
+ values[4] |= static_cast<uint64_t>(*ptr++) << 27;
+ values[4] |= *ptr++ << 19;
+ values[4] |= *ptr++ << 11;
+ values[4] |= *ptr++ << 3;
+ values[4] |= *ptr >> 5;
+
+ values[5] = static_cast<uint64_t>(*ptr++ & 0x1f) << 58;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 50;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 42;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 34;
+ values[5] |= static_cast<uint64_t>(*ptr++) << 26;
+ values[5] |= *ptr++ << 18;
+ values[5] |= *ptr++ << 10;
+ values[5] |= *ptr++ << 2;
+ values[5] |= *ptr >> 6;
+
+ values[6] = static_cast<uint64_t>(*ptr++ & 0x3f) << 57;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 49;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 41;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 33;
+ values[6] |= static_cast<uint64_t>(*ptr++) << 25;
+ values[6] |= *ptr++ << 17;
+ values[6] |= *ptr++ << 9;
+ values[6] |= *ptr++ << 1;
+ values[6] |= *ptr >> 7;
+
+ values[7] = static_cast<uint64_t>(*ptr++ & 0x7f) << 56;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 48;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 40;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 32;
+ values[7] |= static_cast<uint64_t>(*ptr++) << 24;
+ values[7] |= *ptr++ << 16;
+ values[7] |= *ptr++ << 8;
+ values[7] |= *ptr;
+}
+
+static inline void pack_bits_block8(const uint64_t* values, uint8_t* ptr, uint8_t bits) {
+ switch (bits) {
+ case 1: pack_bits_1(values, ptr); break;
+ case 2: pack_bits_2(values, ptr); break;
+ case 3: pack_bits_3(values, ptr); break;
+ case 4: pack_bits_4(values, ptr); break;
+ case 5: pack_bits_5(values, ptr); break;
+ case 6: pack_bits_6(values, ptr); break;
+ case 7: pack_bits_7(values, ptr); break;
+ case 8: pack_bits_8(values, ptr); break;
+ case 9: pack_bits_9(values, ptr); break;
+ case 10: pack_bits_10(values, ptr); break;
+ case 11: pack_bits_11(values, ptr); break;
+ case 12: pack_bits_12(values, ptr); break;
+ case 13: pack_bits_13(values, ptr); break;
+ case 14: pack_bits_14(values, ptr); break;
+ case 15: pack_bits_15(values, ptr); break;
+ case 16: pack_bits_16(values, ptr); break;
+ case 17: pack_bits_17(values, ptr); break;
+ case 18: pack_bits_18(values, ptr); break;
+ case 19: pack_bits_19(values, ptr); break;
+ case 20: pack_bits_20(values, ptr); break;
+ case 21: pack_bits_21(values, ptr); break;
+ case 22: pack_bits_22(values, ptr); break;
+ case 23: pack_bits_23(values, ptr); break;
+ case 24: pack_bits_24(values, ptr); break;
+ case 25: pack_bits_25(values, ptr); break;
+ case 26: pack_bits_26(values, ptr); break;
+ case 27: pack_bits_27(values, ptr); break;
+ case 28: pack_bits_28(values, ptr); break;
+ case 29: pack_bits_29(values, ptr); break;
+ case 30: pack_bits_30(values, ptr); break;
+ case 31: pack_bits_31(values, ptr); break;
+ case 32: pack_bits_32(values, ptr); break;
+ case 33: pack_bits_33(values, ptr); break;
+ case 34: pack_bits_34(values, ptr); break;
+ case 35: pack_bits_35(values, ptr); break;
+ case 36: pack_bits_36(values, ptr); break;
+ case 37: pack_bits_37(values, ptr); break;
+ case 38: pack_bits_38(values, ptr); break;
+ case 39: pack_bits_39(values, ptr); break;
+ case 40: pack_bits_40(values, ptr); break;
+ case 41: pack_bits_41(values, ptr); break;
+ case 42: pack_bits_42(values, ptr); break;
+ case 43: pack_bits_43(values, ptr); break;
+ case 44: pack_bits_44(values, ptr); break;
+ case 45: pack_bits_45(values, ptr); break;
+ case 46: pack_bits_46(values, ptr); break;
+ case 47: pack_bits_47(values, ptr); break;
+ case 48: pack_bits_48(values, ptr); break;
+ case 49: pack_bits_49(values, ptr); break;
+ case 50: pack_bits_50(values, ptr); break;
+ case 51: pack_bits_51(values, ptr); break;
+ case 52: pack_bits_52(values, ptr); break;
+ case 53: pack_bits_53(values, ptr); break;
+ case 54: pack_bits_54(values, ptr); break;
+ case 55: pack_bits_55(values, ptr); break;
+ case 56: pack_bits_56(values, ptr); break;
+ case 57: pack_bits_57(values, ptr); break;
+ case 58: pack_bits_58(values, ptr); break;
+ case 59: pack_bits_59(values, ptr); break;
+ case 60: pack_bits_60(values, ptr); break;
+ case 61: pack_bits_61(values, ptr); break;
+ case 62: pack_bits_62(values, ptr); break;
+ case 63: pack_bits_63(values, ptr); break;
+ default: throw std::logic_error("wrong number of bits " + std::to_string(bits));
+ }
+}
+
+static inline void unpack_bits_block8(uint64_t* values, const uint8_t* ptr, uint8_t bits) {
+ switch (bits) {
+ case 1: unpack_bits_1(values, ptr); break;
+ case 2: unpack_bits_2(values, ptr); break;
+ case 3: unpack_bits_3(values, ptr); break;
+ case 4: unpack_bits_4(values, ptr); break;
+ case 5: unpack_bits_5(values, ptr); break;
+ case 6: unpack_bits_6(values, ptr); break;
+ case 7: unpack_bits_7(values, ptr); break;
+ case 8: unpack_bits_8(values, ptr); break;
+ case 9: unpack_bits_9(values, ptr); break;
+ case 10: unpack_bits_10(values, ptr); break;
+ case 11: unpack_bits_11(values, ptr); break;
+ case 12: unpack_bits_12(values, ptr); break;
+ case 13: unpack_bits_13(values, ptr); break;
+ case 14: unpack_bits_14(values, ptr); break;
+ case 15: unpack_bits_15(values, ptr); break;
+ case 16: unpack_bits_16(values, ptr); break;
+ case 17: unpack_bits_17(values, ptr); break;
+ case 18: unpack_bits_18(values, ptr); break;
+ case 19: unpack_bits_19(values, ptr); break;
+ case 20: unpack_bits_20(values, ptr); break;
+ case 21: unpack_bits_21(values, ptr); break;
+ case 22: unpack_bits_22(values, ptr); break;
+ case 23: unpack_bits_23(values, ptr); break;
+ case 24: unpack_bits_24(values, ptr); break;
+ case 25: unpack_bits_25(values, ptr); break;
+ case 26: unpack_bits_26(values, ptr); break;
+ case 27: unpack_bits_27(values, ptr); break;
+ case 28: unpack_bits_28(values, ptr); break;
+ case 29: unpack_bits_29(values, ptr); break;
+ case 30: unpack_bits_30(values, ptr); break;
+ case 31: unpack_bits_31(values, ptr); break;
+ case 32: unpack_bits_32(values, ptr); break;
+ case 33: unpack_bits_33(values, ptr); break;
+ case 34: unpack_bits_34(values, ptr); break;
+ case 35: unpack_bits_35(values, ptr); break;
+ case 36: unpack_bits_36(values, ptr); break;
+ case 37: unpack_bits_37(values, ptr); break;
+ case 38: unpack_bits_38(values, ptr); break;
+ case 39: unpack_bits_39(values, ptr); break;
+ case 40: unpack_bits_40(values, ptr); break;
+ case 41: unpack_bits_41(values, ptr); break;
+ case 42: unpack_bits_42(values, ptr); break;
+ case 43: unpack_bits_43(values, ptr); break;
+ case 44: unpack_bits_44(values, ptr); break;
+ case 45: unpack_bits_45(values, ptr); break;
+ case 46: unpack_bits_46(values, ptr); break;
+ case 47: unpack_bits_47(values, ptr); break;
+ case 48: unpack_bits_48(values, ptr); break;
+ case 49: unpack_bits_49(values, ptr); break;
+ case 50: unpack_bits_50(values, ptr); break;
+ case 51: unpack_bits_51(values, ptr); break;
+ case 52: unpack_bits_52(values, ptr); break;
+ case 53: unpack_bits_53(values, ptr); break;
+ case 54: unpack_bits_54(values, ptr); break;
+ case 55: unpack_bits_55(values, ptr); break;
+ case 56: unpack_bits_56(values, ptr); break;
+ case 57: unpack_bits_57(values, ptr); break;
+ case 58: unpack_bits_58(values, ptr); break;
+ case 59: unpack_bits_59(values, ptr); break;
+ case 60: unpack_bits_60(values, ptr); break;
+ case 61: unpack_bits_61(values, ptr); break;
+ case 62: unpack_bits_62(values, ptr); break;
+ case 63: unpack_bits_63(values, ptr); break;
+ default: throw std::logic_error("wrong number of bits " + std::to_string(bits));
+ }
+}
+
+} // namespace
+
+#endif // BIT_PACKING_HPP_
diff --git a/theta/include/compact_theta_sketch_parser.hpp b/theta/include/compact_theta_sketch_parser.hpp
index a06924e..8a88324 100644
--- a/theta/include/compact_theta_sketch_parser.hpp
+++ b/theta/include/compact_theta_sketch_parser.hpp
@@ -20,7 +20,7 @@
#ifndef COMPACT_THETA_SKETCH_PARSER_HPP_
#define COMPACT_THETA_SKETCH_PARSER_HPP_
-#include <stdint.h>
+#include <cstdint>
namespace datasketches {
@@ -33,7 +33,8 @@ public:
uint16_t seed_hash;
uint32_t num_entries;
uint64_t theta;
- const uint64_t* entries;
+ const void* entries_start_ptr;
+ uint8_t entry_bits;
};
static compact_theta_sketch_data parse(const void* ptr, size_t size, uint64_t seed, bool dump_on_error = false);
@@ -45,18 +46,24 @@ private:
static const size_t COMPACT_SKETCH_TYPE_BYTE = 2;
static const size_t COMPACT_SKETCH_FLAGS_BYTE = 5;
static const size_t COMPACT_SKETCH_SEED_HASH_U16 = 3;
- static const size_t COMPACT_SKETCH_NUM_ENTRIES_U32 = 2;
- static const size_t COMPACT_SKETCH_SINGLE_ENTRY_U64 = 1;
- static const size_t COMPACT_SKETCH_ENTRIES_EXACT_U64 = 2;
- static const size_t COMPACT_SKETCH_THETA_U64 = 2;
- static const size_t COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 = 3;
+ static const size_t COMPACT_SKETCH_SINGLE_ENTRY_U64 = 1; // ver 3
+ static const size_t COMPACT_SKETCH_NUM_ENTRIES_U32 = 2; // ver 1-3
+ static const size_t COMPACT_SKETCH_ENTRIES_EXACT_U64 = 2; // ver 1-3
+ static const size_t COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 = 3; // ver 1-3
+ static const size_t COMPACT_SKETCH_THETA_U64 = 2; // ver 1-3
+ static const size_t COMPACT_SKETCH_V4_MIN_ENTRY_ZEROS_BYTE = 3;
+ static const size_t COMPACT_SKETCH_V4_THETA_U64 = 1;
+ static const size_t COMPACT_SKETCH_V4_NUM_ENTRIES_EXACT_U32 = 2;
+ static const size_t COMPACT_SKETCH_V4_NUM_ENTRIES_ESTIMATION_U32 = 4;
+ static const size_t COMPACT_SKETCH_V4_PACKED_DATA_EXACT_U8 = 12;
+ static const size_t COMPACT_SKETCH_V4_PACKED_DATA_ESTIMATION_U8 = 20;
static const uint8_t COMPACT_SKETCH_IS_EMPTY_FLAG = 2;
static const uint8_t COMPACT_SKETCH_IS_ORDERED_FLAG = 4;
- static const uint8_t COMPACT_SKETCH_SERIAL_VERSION = 3;
static const uint8_t COMPACT_SKETCH_TYPE = 3;
+ static void check_memory_size(const void* ptr, size_t actual_bytes, size_t expected_bytes, bool dump_on_error);
static std::string hex_dump(const uint8_t* ptr, size_t size);
};
diff --git a/theta/include/compact_theta_sketch_parser_impl.hpp b/theta/include/compact_theta_sketch_parser_impl.hpp
index 4b653b8..115cdda 100644
--- a/theta/include/compact_theta_sketch_parser_impl.hpp
+++ b/theta/include/compact_theta_sketch_parser_impl.hpp
@@ -28,104 +28,107 @@ namespace datasketches {
template<bool dummy>
auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed, bool dump_on_error) -> compact_theta_sketch_data {
- if (size < 8) throw std::out_of_range("at least 8 bytes expected, actual " + std::to_string(size)
- + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
-
+ check_memory_size(ptr, size, 8, dump_on_error);
+ checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
uint8_t serial_version = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE];
-
switch(serial_version) {
- case COMPACT_SKETCH_SERIAL_VERSION: {
- checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ case 4: {
+ // version 4 sketches are ordered and always have entries (single item in exact mode is v3)
+ const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+ checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 1;
+ uint64_t theta = theta_constants::MAX_THETA;
+ if (has_theta) {
+ check_memory_size(ptr, size, 16, dump_on_error);
+ theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_V4_THETA_U64];
+ }
+ const size_t num_entries_index = has_theta ? COMPACT_SKETCH_V4_NUM_ENTRIES_ESTIMATION_U32 : COMPACT_SKETCH_V4_NUM_ENTRIES_EXACT_U32;
+ check_memory_size(ptr, size, (num_entries_index + 1) * sizeof(uint32_t), dump_on_error);
+ const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[num_entries_index];
+ const size_t entries_offset_bytes = has_theta ? COMPACT_SKETCH_V4_PACKED_DATA_ESTIMATION_U8 : COMPACT_SKETCH_V4_PACKED_DATA_EXACT_U8;
+ const uint8_t min_entry_zeros = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_V4_MIN_ENTRY_ZEROS_BYTE];
+ const size_t expected_bits = (64 - min_entry_zeros) * num_entries;
+ const size_t expected_size_bytes = entries_offset_bytes + std::ceil(expected_bits / 8.0);
+ check_memory_size(ptr, size, expected_size_bytes, dump_on_error);
+ return {false, true, seed_hash, num_entries, theta,
+ reinterpret_cast<const uint8_t*>(ptr) + entries_offset_bytes, static_cast<uint8_t>(64 - min_entry_zeros)};
+ }
+ case 3: {
uint64_t theta = theta_constants::MAX_THETA;
const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
- return {true, true, seed_hash, 0, theta, nullptr};
+ return {true, true, seed_hash, 0, theta, nullptr, 64};
}
checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
if (has_theta) {
- if (size < 16) throw std::out_of_range("at least 16 bytes expected, actual " + std::to_string(size));
+ check_memory_size(ptr, size, (COMPACT_SKETCH_THETA_U64 + 1) * sizeof(uint64_t), dump_on_error);
theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
}
if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
- if (size < 16) throw std::out_of_range("at least 16 bytes expected, actual " + std::to_string(size));
- return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+ check_memory_size(ptr, size, 16, dump_on_error);
+ return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64, 64};
}
const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
- if (size < expected_size_bytes) {
- throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
- + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
- }
+ check_memory_size(ptr, size, expected_size_bytes, dump_on_error);
const bool is_ordered = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_ORDERED_FLAG);
- return {false, is_ordered, seed_hash, num_entries, theta, entries};
+ return {false, is_ordered, seed_hash, num_entries, theta, entries, 64};
}
case 1: {
uint16_t seed_hash = compute_seed_hash(seed);
- checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
- if (is_empty) {
- return {true, true, seed_hash, 0, theta, nullptr};
- }
+ if (is_empty) return {true, true, seed_hash, 0, theta, nullptr, 64};
const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
- if (size < expected_size_bytes) {
- throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
- + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
- }
- return {false, true, seed_hash, num_entries, theta, entries};
+ check_memory_size(ptr, size, expected_size_bytes, dump_on_error);
+ return {false, true, seed_hash, num_entries, theta, entries, 64};
}
case 2: {
- uint8_t preamble_size = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE];
- checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+ uint8_t preamble_size = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE];
const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
if (preamble_size == 1) {
- return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+ return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr, 64};
} else if (preamble_size == 2) {
const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
if (num_entries == 0) {
- return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+ return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr, 64};
} else {
const size_t expected_size_bytes = (preamble_size + num_entries) << 3;
- if (size < expected_size_bytes) {
- throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
- + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
- }
+ check_memory_size(ptr, size, expected_size_bytes, dump_on_error);
const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_EXACT_U64;
- return {false, true, seed_hash, num_entries, theta_constants::MAX_THETA, entries};
+ return {false, true, seed_hash, num_entries, theta_constants::MAX_THETA, entries, 64};
}
} else if (preamble_size == 3) {
const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
- if (is_empty) {
- return {true, true, seed_hash, 0, theta, nullptr};
- }
+ if (is_empty) return {true, true, seed_hash, 0, theta, nullptr, 64};
const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
- if (size < expected_size_bytes) {
- throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
- + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
- }
- return {false, true, seed_hash, num_entries, theta, entries};
+ check_memory_size(ptr, size, expected_size_bytes, dump_on_error);
+ return {false, true, seed_hash, num_entries, theta, entries, 64};
} else {
throw std::invalid_argument(std::to_string(preamble_size) + " longs of premable, but expected 1, 2, or 3");
}
}
default:
- // this should always fail since the valid cases are handled above
- checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
- // this throw is never reached, because check_serial_version will throw an informative exception.
- // This is only here to avoid a compiler warning about a path without a return value.
- throw std::invalid_argument("unexpected sketch serialization version");
+ throw std::invalid_argument("unsupported serial version " + std::to_string(serial_version));
}
}
+template<bool dummy>
+void compact_theta_sketch_parser<dummy>::check_memory_size(const void* ptr, size_t actual_bytes, size_t expected_bytes, bool dump_on_error) {
+ if (actual_bytes < expected_bytes) throw std::out_of_range("at least " + std::to_string(expected_bytes)
+ + " bytes expected, actual " + std::to_string(actual_bytes)
+ + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), actual_bytes)) : ""));
+}
+
template<bool dummy>
std::string compact_theta_sketch_parser<dummy>::hex_dump(const uint8_t* ptr, size_t size) {
std::stringstream s;
diff --git a/theta/include/theta_sketch.hpp b/theta/include/theta_sketch.hpp
index fa283b8..e638f93 100644
--- a/theta/include/theta_sketch.hpp
+++ b/theta/include/theta_sketch.hpp
@@ -21,6 +21,7 @@
#define THETA_SKETCH_HPP_
#include "theta_update_sketch_base.hpp"
+#include "compact_theta_sketch_parser.hpp"
namespace datasketches {
@@ -355,6 +356,8 @@ public:
*/
vector_bytes serialize(unsigned header_size_bytes = 0) const;
+ vector_bytes serialize_compressed(unsigned header_size_bytes = 0) const;
+
virtual iterator begin();
virtual iterator end();
virtual const_iterator begin() const;
@@ -391,6 +394,14 @@ private:
uint64_t theta_;
std::vector<uint64_t, Allocator> entries_;
+ vector_bytes serialize_version_4_SLZ(unsigned header_size_bytes = 0) const;
+ vector_bytes serialize_version_4_MLZ(unsigned header_size_bytes = 0) const;
+ vector_bytes serialize_version_4_FLZ(unsigned header_size_bytes = 0) const;
+ vector_bytes serialize_version_4_ULEB128(unsigned header_size_bytes = 0) const;
+
+ static compact_theta_sketch_alloc deserialize_version_4(const void* bytes, size_t size,
+ uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
virtual void print_specifics(std::ostringstream& os) const;
};
@@ -407,7 +418,7 @@ public:
template<typename Allocator = std::allocator<uint64_t>>
class wrapped_compact_theta_sketch_alloc : public base_theta_sketch_alloc<Allocator> {
public:
- using const_iterator = const uint64_t*;
+ class const_iterator;
Allocator get_allocator() const;
bool is_empty() const;
@@ -433,15 +444,32 @@ protected:
virtual void print_items(std::ostringstream& os) const;
private:
- bool is_empty_;
- bool is_ordered_;
- uint16_t seed_hash_;
- uint32_t num_entries_;
- uint64_t theta_;
- const uint64_t* entries_;
+ using data_type = compact_theta_sketch_parser<true>::compact_theta_sketch_data;
+ data_type data_;
+
+ wrapped_compact_theta_sketch_alloc(const data_type& data);
+};
- wrapped_compact_theta_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint32_t num_entries,
- uint64_t theta, const uint64_t* entries);
+template<typename Allocator>
+class wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator: std::iterator<std::input_iterator_tag, uint64_t> {
+public:
+ const_iterator(const void* ptr, uint8_t entry_bits, uint32_t num_entries, uint32_t index);
+ const_iterator& operator++();
+ const_iterator operator++(int);
+ bool operator==(const const_iterator& other) const;
+ bool operator!=(const const_iterator& other) const;
+ const uint64_t& operator*() const;
+ const uint64_t* operator->() const;
+private:
+ const void* ptr_;
+ uint8_t entry_bits_;
+ uint32_t num_entries_;
+ uint32_t index_;
+ uint64_t previous_;
+ bool is_block_mode_;
+ uint8_t buf_i_;
+ uint8_t offset_;
+ uint64_t buffer_[8];
};
// aliases with default allocator for convenience
diff --git a/theta/include/theta_sketch_impl.hpp b/theta/include/theta_sketch_impl.hpp
index 2e06b89..a889b57 100644
--- a/theta/include/theta_sketch_impl.hpp
+++ b/theta/include/theta_sketch_impl.hpp
@@ -27,7 +27,8 @@
#include "serde.hpp"
#include "binomial_bounds.hpp"
#include "theta_helpers.hpp"
-#include "compact_theta_sketch_parser.hpp"
+#include "count_zeros.hpp"
+#include "bit_packing.hpp"
namespace datasketches {
@@ -401,6 +402,239 @@ auto compact_theta_sketch_alloc<A>::serialize(unsigned header_size_bytes) const
return bytes;
}
+template<typename A>
+auto compact_theta_sketch_alloc<A>::serialize_compressed(unsigned header_size_bytes) const -> vector_bytes {
+ if (!this->is_ordered() || entries_.size() == 0 ||
+ (entries_.size() == 1 && !this->is_estimation_mode())) return serialize(header_size_bytes);
+ return serialize_version_4_MLZ(header_size_bytes);
+}
+
+// should be called for ordered sketches that are not empty and not single item
+template<typename A>
+auto compact_theta_sketch_alloc<A>::serialize_version_4_FLZ(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
+ // compression based on leading zeros in deltas between ordered hash values
+ // assumes ordered sketch
+ uint64_t previous = 0;
+ uint8_t min_zeros = 64;
+ uint8_t max_zeros = 0;
+ std::vector<uint8_t> leading_zeros(entries_.get_allocator());
+ leading_zeros.reserve(entries_.size());
+ std::vector<uint64_t> deltas(entries_.get_allocator());
+ deltas.reserve(entries_.size());
+ size_t compressed_bits = 0;
+ for (unsigned i = 0; i < entries_.size(); ++i) {
+ const uint64_t delta = entries_[i] - previous;
+ deltas.push_back(delta);
+ previous = entries_[i];
+ const uint8_t zeros = count_leading_zeros_in_u64(delta);
+ leading_zeros.push_back(zeros);
+ min_zeros = std::min(min_zeros, zeros);
+ max_zeros = std::max(max_zeros, zeros);
+ compressed_bits += 63 - zeros; // the first 1 is understood
+ }
+ const uint8_t count_zeros_bits = 8 - byte_leading_zeros_table[max_zeros - min_zeros];
+ compressed_bits += count_zeros_bits * entries_.size();
+
+ const uint32_t num_entries = static_cast<uint32_t>(entries_.size());
+ const uint8_t num_entries_bits = 31 - count_leading_zeros_in_u32(num_entries); // the first 1 is understood
+ compressed_bits += 5 + num_entries_bits;
+
+ const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs + std::ceil(compressed_bits / 8.0);
+ vector_bytes bytes(size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(preamble_longs, ptr);
+ const uint8_t serial_version = 4;
+ ptr += copy_to_mem(serial_version, ptr);
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(type, ptr);
+ ptr += copy_to_mem(min_zeros, ptr);
+ ptr += copy_to_mem(count_zeros_bits, ptr);
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(flags_byte, ptr);
+ const uint16_t seed_hash = get_seed_hash();
+ ptr += copy_to_mem(seed_hash, ptr);
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(theta_, ptr);
+ }
+
+ size_t offset_bits = pack_bits(num_entries_bits, 5, ptr, 0);
+ offset_bits = pack_bits(num_entries, num_entries_bits, ptr, offset_bits);
+ for (unsigned i = 0; i < entries_.size(); ++i) {
+ offset_bits = pack_bits(leading_zeros[i] - min_zeros, count_zeros_bits, ptr, offset_bits);
+ offset_bits = pack_bits(deltas[i], 63 - leading_zeros[i], ptr, offset_bits);
+ }
+ return bytes;
+}
+
+template<typename A>
+auto compact_theta_sketch_alloc<A>::serialize_version_4_MLZ(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
+ // compression based on leading zeros in deltas between ordered hash values
+ // assumes ordered sketch
+ uint64_t previous = 0;
+ uint64_t ored = 0;
+ for (const uint64_t entry: entries_) {
+ const uint64_t delta = entry - previous;
+ ored |= delta;
+ previous = entry;
+ }
+ uint8_t min_entry_zeros = count_leading_zeros_in_u64(ored);
+ size_t compressed_bits = (64 - min_entry_zeros) * entries_.size();
+
+// const uint8_t num_entries_zeros = count_leading_zeros_in_u32(entries_.size());
+// const uint8_t num_entries_bits = 31 - num_entries_zeros; // first 1 is understood
+// compressed_bits += num_entries_bits * (entries_.size() > 1); // no bits for 0 or 1 entry
+
+ const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs + std::ceil(compressed_bits / 8.0) + sizeof(uint32_t);
+ vector_bytes bytes(size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(preamble_longs, ptr);
+ const uint8_t serial_version = 4;
+ ptr += copy_to_mem<uint8_t>(serial_version, ptr);
+ ptr += copy_to_mem(SKETCH_TYPE, ptr);
+ ptr += copy_to_mem(min_entry_zeros, ptr);
+ ptr += sizeof(uint8_t); // unused
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(flags_byte, ptr);
+ ptr += copy_to_mem<uint16_t>(get_seed_hash(), ptr);
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(theta_, ptr);
+ }
+// uint8_t offset_bits = 0;
+// if (entries_.size() > 1) {
+// offset_bits = put_bits64(entries_.size(), num_entries_bits, ptr, offset_bits);
+// std::cout << "writing " << std::to_string(num_entries_bits) << " bits\n";
+// }
+
+ // uncompressed num_entries for now
+ ptr += copy_to_mem<uint32_t>(entries_.size(), ptr);
+
+ const uint8_t entry_bits = 64 - min_entry_zeros;
+
+ previous = 0;
+ uint64_t deltas[8];
+
+ unsigned i;
+ for (i = 0; i + 7 < entries_.size(); i += 8) {
+ for (unsigned j = 0; j < 8; ++j) {
+ deltas[j] = entries_[i + j] - previous;
+ previous = entries_[i + j];
+ }
+ pack_bits_block8(deltas, ptr, entry_bits);
+ ptr += entry_bits;
+ }
+
+ uint8_t offset = 0;
+ for (; i < entries_.size(); ++i) {
+ const uint64_t delta = entries_[i] - previous;
+ previous = entries_[i];
+ offset = pack_bits(delta, entry_bits, ptr, offset);
+ }
+ return bytes;
+}
+
+template<typename A>
+auto compact_theta_sketch_alloc<A>::serialize_version_4_SLZ(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
+ // compression based on leading zeros in deltas between ordered hash values
+ // assumes ordered sketch
+ const size_t fixed_size = header_size_bytes + sizeof(uint64_t) * preamble_longs;
+ const size_t estimated_variable_size = 5 // for num_entries
+ + 9 * entries_.size();
+ vector_bytes bytes(fixed_size + estimated_variable_size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(preamble_longs, ptr);
+ const uint8_t serial_version = 4;
+ ptr += copy_to_mem(serial_version, ptr);
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(type, ptr);
+ ptr += sizeof(uint16_t); // unused
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(flags_byte, ptr);
+ const uint16_t seed_hash = get_seed_hash();
+ ptr += copy_to_mem(seed_hash, ptr);
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(theta_, ptr);
+ }
+
+ ptr += ULEB128(entries_.size(), ptr);
+
+ uint8_t offset_bits = 0;
+ uint64_t previous = 0;
+ for (const uint64_t entry: entries_) {
+ const uint64_t delta = entry - previous;
+ previous = entry;
+ const uint8_t zeros = count_leading_zeros_in_u64(delta);
+ offset_bits = pack_bits(zeros, 6, ptr, offset_bits);
+ offset_bits = pack_bits(delta, 63 - zeros, ptr, offset_bits);
+ }
+ if (offset_bits > 0) ++ptr;
+ bytes.resize(ptr - bytes.data());
+ return bytes;
+}
+
+template<typename A>
+auto compact_theta_sketch_alloc<A>::serialize_version_4_ULEB128(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
+ // compression based on ULEB128 code of deltas between ordered hash values
+ // assumes ordered sketch
+
+ const size_t fixed_size = header_size_bytes + sizeof(uint64_t) * preamble_longs;
+ const size_t estimated_variable_size = 5 // for num_entries
+ + 10 * entries_.size();
+ vector_bytes bytes(fixed_size + estimated_variable_size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(preamble_longs, ptr);
+ const uint8_t serial_version = 4;
+ ptr += copy_to_mem(serial_version, ptr);
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(type, ptr);
+ ptr += sizeof(uint16_t); // unused
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(flags_byte, ptr);
+ const uint16_t seed_hash = get_seed_hash();
+ ptr += copy_to_mem(seed_hash, ptr);
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(theta_, ptr);
+ }
+
+ ptr += ULEB128(entries_.size(), ptr);
+
+ uint64_t previous = 0;
+ for (unsigned i = 0; i < entries_.size(); ++i) {
+ const uint64_t delta = entries_[i] - previous;
+ previous = entries_[i];
+ ptr += pack_ULEB128(delta, ptr);
+ }
+ bytes.resize(ptr - bytes.data());
+ return bytes;
+}
+
template<typename A>
compact_theta_sketch_alloc<A> compact_theta_sketch_alloc<A>::deserialize(std::istream& is, uint64_t seed, const A& allocator) {
const auto preamble_longs = read<uint8_t>(is);
@@ -505,27 +739,93 @@ compact_theta_sketch_alloc<A> compact_theta_sketch_alloc<A>::deserialize(std::is
template<typename A>
compact_theta_sketch_alloc<A> compact_theta_sketch_alloc<A>::deserialize(const void* bytes, size_t size, uint64_t seed, const A& allocator) {
+ if (static_cast<const uint8_t*>(bytes)[1] == 4) return deserialize_version_4(bytes, size, seed, allocator); // TODO: check size
auto data = compact_theta_sketch_parser<true>::parse(bytes, size, seed, false);
- return compact_theta_sketch_alloc(data.is_empty, data.is_ordered, data.seed_hash, data.theta, std::vector<uint64_t, A>(data.entries, data.entries + data.num_entries, allocator));
+ const uint64_t* entries = reinterpret_cast<const uint64_t*>(data.entries_start_ptr);
+ return compact_theta_sketch_alloc(data.is_empty, data.is_ordered, data.seed_hash, data.theta,
+ std::vector<uint64_t, A>(entries, entries + data.num_entries, allocator));
+}
+
+// MLZ
+template<typename A>
+compact_theta_sketch_alloc<A> compact_theta_sketch_alloc<A>::deserialize_version_4(const void* bytes, size_t size,
+ uint64_t seed, const A& allocator) {
+ ensure_minimum_memory(size, 8);
+ const uint8_t* ptr = static_cast<const uint8_t*>(bytes);
+ uint8_t preamble_longs;
+ ptr += copy_from_mem(ptr, preamble_longs);
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, serial_version);
+ uint8_t type;
+ ptr += copy_from_mem(ptr, type);
+ uint8_t min_entry_zeros;
+ ptr += copy_from_mem(ptr, min_entry_zeros);
+// uint8_t num_entries_zeros;
+// ptr += copy_from_mem(ptr, num_entries_zeros);
+ ptr++; // unused
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, flags_byte);
+ uint16_t seed_hash;
+ ptr += copy_from_mem(ptr, seed_hash);
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ checker<true>::check_serial_version(serial_version, 4);
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ if (!is_empty) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+ // check counts of zeros?
+
+ uint64_t theta = theta_constants::MAX_THETA;
+ if (preamble_longs == 2) {
+ ensure_minimum_memory(size, sizeof(uint64_t) * 2);
+ ptr += copy_from_mem(ptr, theta);
+ }
+
+ // TODO: check size
+
+// const size_t expected_bits = (64 - min_entry_zeros) * num_entries + num_entries_bits * (num_entries > 1);
+// const size_t expected_size = sizeof(uint64_t) * (preamble_longs + std::ceil(expected_bits / 64.0));
+// ensure_minimum_memory(size, expected_size);
+
+ // uncompressed num_entries for now
+ uint32_t num_entries;
+ ptr += copy_from_mem(ptr, num_entries);
+
+ const size_t expected_bits = (64 - min_entry_zeros) * num_entries;
+ const size_t expected_size = sizeof(uint64_t) * preamble_longs + std::ceil(expected_bits / 8.0) + sizeof(uint32_t);
+ ensure_minimum_memory(size, expected_size);
+
+ const uint8_t entry_bits = 64 - min_entry_zeros;
+ uint64_t previous = 0;
+ std::vector<uint64_t, A> entries(num_entries, 0, allocator);
+
+ unsigned i;
+ for (i = 0; i + 7 < num_entries; i += 8) {
+ unpack_bits_block8(&entries[i], ptr, entry_bits);
+ ptr += entry_bits;
+ }
+ uint8_t offset = 0;
+ for (; i < num_entries; ++i) {
+ offset = unpack_bits(entries[i], entry_bits, ptr, offset);
+ }
+ // undo deltas
+ for (i = 0; i < num_entries; ++i) {
+ entries[i] += previous;
+ previous = entries[i];
+ }
+
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_theta_sketch_alloc(is_empty, is_ordered, seed_hash, theta, std::move(entries));
}
// wrapped compact sketch
template<typename A>
-wrapped_compact_theta_sketch_alloc<A>::wrapped_compact_theta_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint32_t num_entries,
- uint64_t theta, const uint64_t* entries):
-is_empty_(is_empty),
-is_ordered_(is_ordered),
-seed_hash_(seed_hash),
-num_entries_(num_entries),
-theta_(theta),
-entries_(entries)
+wrapped_compact_theta_sketch_alloc<A>::wrapped_compact_theta_sketch_alloc(const data_type& data):
+data_(data)
{}
template<typename A>
const wrapped_compact_theta_sketch_alloc<A> wrapped_compact_theta_sketch_alloc<A>::wrap(const void* bytes, size_t size, uint64_t seed, bool dump_on_error) {
- auto data = compact_theta_sketch_parser<true>::parse(bytes, size, seed, dump_on_error);
- return wrapped_compact_theta_sketch_alloc(data.is_empty, data.is_ordered, data.seed_hash, data.num_entries, data.theta, data.entries);
+ return wrapped_compact_theta_sketch_alloc(compact_theta_sketch_parser<true>::parse(bytes, size, seed, dump_on_error));
}
template<typename A>
@@ -535,37 +835,37 @@ A wrapped_compact_theta_sketch_alloc<A>::get_allocator() const {
template<typename A>
bool wrapped_compact_theta_sketch_alloc<A>::is_empty() const {
- return is_empty_;
+ return data_.is_empty;
}
template<typename A>
bool wrapped_compact_theta_sketch_alloc<A>::is_ordered() const {
- return is_ordered_;
+ return data_.is_ordered;
}
template<typename A>
uint64_t wrapped_compact_theta_sketch_alloc<A>::get_theta64() const {
- return theta_;
+ return data_.theta;
}
template<typename A>
uint32_t wrapped_compact_theta_sketch_alloc<A>::get_num_retained() const {
- return static_cast<uint32_t>(num_entries_);
+ return data_.num_entries;
}
template<typename A>
uint16_t wrapped_compact_theta_sketch_alloc<A>::get_seed_hash() const {
- return seed_hash_;
+ return data_.seed_hash;
}
template<typename A>
auto wrapped_compact_theta_sketch_alloc<A>::begin() const -> const_iterator {
- return entries_;
+ return const_iterator(data_.entries_start_ptr, data_.entry_bits, data_.num_entries, 0);
}
template<typename A>
auto wrapped_compact_theta_sketch_alloc<A>::end() const -> const_iterator {
- return entries_ + num_entries_;
+ return const_iterator(data_.entries_start_ptr, data_.entry_bits, data_.num_entries, data_.num_entries);
}
template<typename A>
@@ -574,12 +874,109 @@ void wrapped_compact_theta_sketch_alloc<A>::print_specifics(std::ostringstream&)
template<typename A>
void wrapped_compact_theta_sketch_alloc<A>::print_items(std::ostringstream& os) const {
os << "### Retained entries" << std::endl;
- for (const auto& hash: *this) {
+ for (const auto hash: *this) {
os << hash << std::endl;
}
os << "### End retained entries" << std::endl;
}
+// assumes index == 0 or index == num_entries
+template<typename Allocator>
+wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::const_iterator(
+ const void* ptr, uint8_t entry_bits, uint32_t num_entries, uint32_t index):
+ptr_(ptr),
+entry_bits_(entry_bits),
+num_entries_(num_entries),
+index_(index),
+previous_(0),
+is_block_mode_(num_entries_ >= 8),
+buf_i_(0),
+offset_(0)
+{
+ if (entry_bits == 64) { // no compression
+ ptr_ = reinterpret_cast<const uint64_t*>(ptr) + index;
+ } else if (index < num_entries) {
+ if (is_block_mode_) {
+ unpack_bits_block8(buffer_, reinterpret_cast<const uint8_t*>(ptr_), entry_bits_);
+ ptr_ = reinterpret_cast<const uint8_t*>(ptr_) + entry_bits_;
+ for (int i = 0; i < 8; ++i) {
+ buffer_[i] += previous_;
+ previous_ = buffer_[i];
+ }
+ } else {
+ offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
+ buffer_[0] += previous_;
+ previous_ = buffer_[0];
+ }
+ }
+}
+
+template<typename Allocator>
+auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator++() -> const_iterator& {
+ if (entry_bits_ == 64) { // no compression
+ ptr_ = reinterpret_cast<const uint64_t*>(ptr_) + 1;
+ return *this;
+ }
+ ++index_;
+ if (index_ < num_entries_) {
+ if (is_block_mode_) {
+ ++buf_i_;
+ if (buf_i_ == 8) {
+ buf_i_ = 0;
+ if (index_ + 8 < num_entries_) {
+ unpack_bits_block8(buffer_, reinterpret_cast<const uint8_t*>(ptr_), entry_bits_);
+ ptr_ = reinterpret_cast<const uint8_t*>(ptr_) + entry_bits_;
+ for (int i = 0; i < 8; ++i) {
+ buffer_[i] += previous_;
+ previous_ = buffer_[i];
+ }
+ } else {
+ is_block_mode_ = false;
+ offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
+ buffer_[0] += previous_;
+ previous_ = buffer_[0];
+ }
+ }
+ } else {
+ offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
+ buffer_[0] += previous_;
+ previous_ = buffer_[0];
+ }
+ }
+ return *this;
+}
+
+template<typename Allocator>
+auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator++(int) -> const_iterator {
+ const_iterator tmp(*this);
+ operator++();
+ return tmp;
+}
+
+template<typename Allocator>
+bool wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator!=(const const_iterator& other) const {
+ if (entry_bits_ == 64) return ptr_ != other.ptr_;
+ return index_ != other.index_;
+}
+
+template<typename Allocator>
+bool wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator==(const const_iterator& other) const {
+ if (entry_bits_ == 64) return ptr_ == other.ptr_;
+ return index_ == other.index_;
+}
+
+template<typename Allocator>
+const uint64_t& wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator*() const {
+ if (entry_bits_ == 64) return *reinterpret_cast<const uint64_t*>(ptr_);
+ return buffer_[buf_i_];
+}
+
+template<typename Allocator>
+const uint64_t* wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator->() const {
+ if (entry_bits_ == 64) return reinterpret_cast<const uint64_t*>(ptr_);
+ return buffer_ + buf_i_;
+}
+
} /* namespace datasketches */
#endif
diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp
index 4fc6651..f57e0e5 100644
--- a/theta/test/theta_sketch_test.cpp
+++ b/theta/test/theta_sketch_test.cpp
@@ -607,7 +607,7 @@ TEST_CASE("theta sketch: wrap compact estimation from java", "[theta_sketch]") {
compact_theta_sketch compact_sketch = update_sketch.compact();
// the sketches are ordered, so the iteration sequence must match exactly
auto iter = sketch.begin();
- for (const auto& key: compact_sketch) {
+ for (const auto key: compact_sketch) {
REQUIRE(*iter == key);
++iter;
}
@@ -652,7 +652,7 @@ TEST_CASE("theta sketch: wrap compact v1 estimation from java", "[theta_sketch]"
compact_theta_sketch compact_sketch = update_sketch.compact();
// the sketches are ordered, so the iteration sequence must match exactly
auto iter = sketch.begin();
- for (const auto& key: compact_sketch) {
+ for (const auto key: compact_sketch) {
REQUIRE(*iter == key);
++iter;
}
@@ -697,7 +697,20 @@ TEST_CASE("theta sketch: wrap compact v2 estimation from java", "[theta_sketch]"
compact_theta_sketch compact_sketch = update_sketch.compact();
// the sketches are ordered, so the iteration sequence must match exactly
auto iter = sketch.begin();
- for (const auto& key: compact_sketch) {
+ for (const auto key: compact_sketch) {
+ REQUIRE(*iter == key);
+ ++iter;
+ }
+}
+
+TEST_CASE("theta sketch: wrapped compressed", "[theta_sketch]") {
+ auto update_sketch = update_theta_sketch::builder().build();
+ for (int i = 0; i < 10000; i++) update_sketch.update(i);
+ auto compact_sketch = update_sketch.compact();
+ auto bytes = compact_sketch.serialize_compressed();
+ auto wrapped_compressed = wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size());
+ auto iter = wrapped_compressed.begin();
+ for (const auto key: compact_sketch) {
REQUIRE(*iter == key);
++iter;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org