You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ai...@apache.org on 2026/10/23 07:56:27 UTC
(doris-thirdparty) branch clucene-3.1 updated: [refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#422)
This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch clucene-3.1
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene-3.1 by this push:
new 80c5e18f2bc [refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#422)
80c5e18f2bc is described below
commit 80c5e18f2bcbadc8ef6187e3c86eafca52ada5bb
Author: Jack <ji...@selectdb.com>
AuthorDate: Fri Oct 23 15:56:20 2026 +0800
[refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#422)
### What problem does this PR solve?
Issue Number: None
Related PR: #421
Problem Summary:
Pick of #421 to clucene-3.1. `src/ext/for` is a vendored copy of powturbo's TurboPFor, which is licensed under GPL v2, linked into clucene-core and thereby into every Doris BE. It is replaced by `CLucene/util/PForCodec.{h,cpp}`, an independent implementation of the same on-disk block format (bit compatible in both directions), and `src/ext/for`, the `ic` library target and the `ic` link dependencies are removed. See #421 for the details, the measurements and the tests of the codec.
Differences to the original change: none, it applies without a conflict apart from the removal of `src/ext/for`.
### Release note
Removed the GPL v2 licensed TurboPFor library from the inverted index code. The index format is unchanged.
### Check List (For Author)
- Test: Unit Test
- Full cl_test (83 tests) built with clang + AVX2 and with g++-11 and the default options (the unit test job of the CI): all passed.
- Behavior changed: No
- Does this need documentation: No
(cherry picked from commit 5725b460b6b12a6874b171832f9f834ce161b2f4)
Co-authored-by: Claude Sonnet 5.5 <no...@anthropic.com>
---
CMakeLists.txt | 2 -
cmake/TurboPFOR.cmake | 15 -
src/core/CLucene/util/PFORUtil.cpp | 314 +-
src/core/CLucene/util/PForCodec.cpp | 846 ++++
src/core/CLucene/util/PForCodec.h | 93 +
src/core/CMakeLists.txt | 2 +-
src/demo/CMakeLists.txt | 20 +-
src/ext/for/CMakeLists.txt | 107 -
src/ext/for/README.md | 585 ---
src/ext/for/bitpack.c | 428 --
src/ext/for/bitpack.h | 327 --
src/ext/for/bitpack_.h | 5190 -------------------
src/ext/for/bitunpack.c | 2433 ---------
src/ext/for/bitunpack_.h | 6049 -----------------------
src/ext/for/bitutil.c | 689 ---
src/ext/for/bitutil.h | 547 --
src/ext/for/conf.h | 282 --
src/ext/for/fp.h | 125 -
src/ext/for/makefile | 156 -
src/ext/for/makefile.vs | 78 -
src/ext/for/sse_neon.h | 355 --
src/ext/for/test_bitd1unpack.cpp | 399 --
src/ext/for/time_.h | 252 -
src/ext/for/transpose.c | 1223 -----
src/ext/for/transpose.h | 113 -
src/ext/for/trle.h | 72 -
src/ext/for/trle_.h | 60 -
src/ext/for/trlec.c | 344 --
src/ext/for/trled.c | 421 --
src/ext/for/v8.c | 1466 ------
src/ext/for/vint.c | 407 --
src/ext/for/vint.h | 401 --
src/ext/for/vp4.h | 357 --
src/ext/for/vp4c.c | 424 --
src/ext/for/vp4d.c | 563 ---
src/ext/for/vs/bitpack_avx2.c | 2 -
src/ext/for/vs/bitpack_sse.c | 2 -
src/ext/for/vs/bitunpack_avx2.c | 2 -
src/ext/for/vs/bitunpack_sse.c | 2 -
src/ext/for/vs/getopt.c | 562 ---
src/ext/for/vs/getopt.h | 97 -
src/ext/for/vs/inttypes.h | 306 --
src/ext/for/vs/stdint.h | 259 -
src/ext/for/vs/transpose_avx2.c | 2 -
src/ext/for/vs/transpose_sse.c | 2 -
src/ext/for/vs/vp4c_avx2.c | 2 -
src/ext/for/vs/vp4c_sse.c | 2 -
src/ext/for/vs/vp4d_avx2.c | 2 -
src/ext/for/vs/vp4d_sse.c | 2 -
src/ext/for/vs/vs2017/TurboPFor.sln | 41 -
src/ext/for/vs/vs2017/TurboPFor.vcxproj | 226 -
src/ext/for/vs/vs2017/TurboPFor.vcxproj.filters | 101 -
src/ext/for/vs/vs2017/icapp.vcxproj | 175 -
src/ext/for/vs/vs2017/icapp.vcxproj.filters | 21 -
src/ext/for/vsimple.c | 537 --
src/ext/for/vsimple.h | 47 -
src/test/CMakeLists.txt | 10 +-
src/test/data/pfor_codec_golden.dat | Bin 0 -> 106756 bytes
src/test/store/testPFOR.cpp | 411 +-
59 files changed, 1412 insertions(+), 26546 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e4cc0af8082..3bcfcb1fd4e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -219,8 +219,6 @@ find_package(Zstd REQUIRED)
#sse2neon
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/3rdparty/sse2neon)
-#TurboPFOR
-ADD_SUBDIRECTORY (src/ext/for)
ADD_SUBDIRECTORY (src/shared)
ADD_SUBDIRECTORY (src/core)
ADD_SUBDIRECTORY (src/test)
diff --git a/cmake/TurboPFOR.cmake b/cmake/TurboPFOR.cmake
deleted file mode 100644
index a742968728c..00000000000
--- a/cmake/TurboPFOR.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-PROJECT(turbo-pfor)
-set(PFOR_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/ext/for)
-
-add_custom_target(build_ic ALL
- COMMAND make OPT='-Wno-implicit-function-declaration -Wno-implicit-int -Wno-int-conversion' USE_AVX2=${USE_AVX2} libic.a -j 8
- WORKING_DIRECTORY ${PFOR_SOURCE_DIR}
- COMMENT "Original Turbo-PFOR makefile target")
-
-add_library(ic STATIC IMPORTED)
-set_target_properties(ic PROPERTIES IMPORTED_LOCATION "${PFOR_SOURCE_DIR}/libic.a")
-add_dependencies(ic build_ic)
-
-install(FILES ${PFOR_SOURCE_DIR}/libic.a
- DESTINATION "lib"
- COMPONENT development)
diff --git a/src/core/CLucene/util/PFORUtil.cpp b/src/core/CLucene/util/PFORUtil.cpp
index 7cc836514ac..e9ce9fe93e7 100644
--- a/src/core/CLucene/util/PFORUtil.cpp
+++ b/src/core/CLucene/util/PFORUtil.cpp
@@ -16,28 +16,19 @@
// under the License.
#include "PFORUtil.h"
-#include "CLucene/debug/error.h"
-#include "CLucene/index/CodeMode.h"
-#include "vp4.h"
-#if (defined(__i386) || defined(__x86_64__))
-#include <cpuid.h>
-#endif
-
#include "CLucene/CLConfig.h"
+#include "CLucene/debug/error.h"
#include "CLucene/index/CodeMode.h"
+#include "CLucene/util/PForCodec.h"
CL_NS_USE(index)
namespace {
-using DEC_FUNC = size_t (*)(unsigned char* __restrict, size_t, uint32_t* __restrict);
-using ENC_FUNC = size_t (*)(uint32_t* __restrict in, size_t n, unsigned char* __restrict out);
-DEC_FUNC g_p4nd1dec;
-DEC_FUNC g_p4nzdec;
-ENC_FUNC g_p4nd1enc;
-ENC_FUNC g_p4nzenc;
-} // namespace
-size_t DefaultDEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict out) {
+// All decoders share this shape: (in, end, n, out) -> bytes consumed.
+using DecodeFn = size_t (*)(const uint8_t*, const uint8_t*, size_t, uint32_t*);
+
+size_t DefaultDEC(const uint8_t* in, const uint8_t* /*end*/, size_t n, uint32_t* out) {
size_t bufferSize = 0;
for (uint32_t i = 0; i < n; i++) {
uint8_t b = in[bufferSize++];
@@ -48,10 +39,10 @@ size_t DefaultDEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict o
}
out[i] = docCode;
}
- return n;
+ return bufferSize;
}
-size_t DefaultDDEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict out) {
+size_t DefaultDDEC(const uint8_t* in, const uint8_t* /*end*/, size_t n, uint32_t* out) {
uint32_t docDelta = 0;
size_t bufferSize = 0;
for (uint32_t i = 0; i < n; i++) {
@@ -61,10 +52,10 @@ size_t DefaultDDEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict
b = in[bufferSize++];
docCode |= (b & 0x7F) << shift;
}
- docDelta += docCode; // Corrected line: Removed right shift
+ docDelta += docCode;
out[i] = docDelta;
}
- return n;
+ return bufferSize;
}
size_t DefaultDENC(uint32_t* __restrict in, size_t n, unsigned char* __restrict out) {
@@ -96,61 +87,126 @@ size_t DefaultENC(uint32_t* __restrict in, size_t n, unsigned char* __restrict o
return outIndex;
}
-__attribute__((constructor)) void SelectPFORFunctions() {
-#if (defined(__i386) || defined(__x86_64__))
- uint32_t eax, ebx, ecx, edx;
- __cpuid(1, eax, ebx, ecx, edx);
-
- bool sse2 = (edx & bit_SSE2) != 0;
- bool sse42 = (ecx & bit_SSE4_2) != 0;
-#if defined(USE_AVX2)
- g_p4nd1dec = p4nd1dec256v32;
- g_p4nzdec = p4nzdec256v32;
- g_p4nd1enc = p4nd1enc256v32;
- g_p4nzenc = p4nzenc256v32;
+// The blocks of the legacy kPfor mode were written by whatever the writing platform
+// used: the 256 value layout on x86_64 with AVX2, the 128 value layout on arm64 and
+// plain variable length integers on x86_64 without AVX2. `compatibleRead` means the
+// index was written by the other platform than the one reading it.
+DecodeFn legacyDocDecoder(bool compatibleRead) {
+#if defined(USE_AVX2) && defined(__AVX2__)
+ return compatibleRead ? lucene::util::pfor::decodeDelta1_128
+ : lucene::util::pfor::decodeDelta1_256;
+#elif defined(__ARM_NEON)
+ return compatibleRead ? lucene::util::pfor::decodeDelta1_256
+ : lucene::util::pfor::decodeDelta1_128;
+#elif defined(__SSSE3__)
+ return compatibleRead ? lucene::util::pfor::decodeDelta1_256 : DefaultDDEC;
#else
- g_p4nd1dec = DefaultDDEC;
- g_p4nzdec = DefaultDEC;
- g_p4nd1enc = DefaultDENC;
- g_p4nzenc = DefaultENC;
+ return DefaultDDEC;
#endif
+}
+
+DecodeFn legacyFreqDecoder(bool compatibleRead) {
+#if defined(USE_AVX2) && defined(__AVX2__)
+ return compatibleRead ? lucene::util::pfor::decodeZigzag128
+ : lucene::util::pfor::decodeZigzag256;
+#elif defined(__ARM_NEON)
+ return compatibleRead ? lucene::util::pfor::decodeZigzag256
+ : lucene::util::pfor::decodeZigzag128;
+#elif defined(__SSSE3__)
+ return compatibleRead ? lucene::util::pfor::decodeZigzag256 : DefaultDEC;
#else
- g_p4nd1dec = p4nd1dec32;
- g_p4nzdec = p4nzdec32;
- g_p4nd1enc = p4nd1enc32;
- g_p4nzenc = p4nzenc32;
+ return DefaultDEC;
#endif
}
+// Scratch space for one serialized array. It is per thread and only grows, so the arrays, which
+// are small and handled one after another, cost neither an allocation nor clearing the buffer.
+std::vector<uint8_t>& scratch(size_t size) {
+ thread_local std::vector<uint8_t> buf;
+ if (buf.size() < size) {
+ buf.resize(size);
+ }
+ return buf;
+}
+
+// Reads a length prefixed serialized array and decodes n values of it. The decoders may look
+// kDecodePadding bytes past the payload.
+void decodeSerialized(IndexInput* in, DecodeFn fn, size_t n, uint32_t* out) {
+ uint32_t size = in->readVInt();
+ std::vector<uint8_t>& buf = scratch(size + lucene::util::pfor::kDecodePadding);
+ in->readBytes(buf.data(), size);
+ fn(buf.data(), buf.data() + size, n, out);
+}
+
+} // namespace
+
size_t P4DEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict out) {
- return g_p4nd1dec(in, n, out);
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ return lucene::util::pfor::decodeDelta1_256(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+#else
+ return DefaultDDEC(in, nullptr, n, out);
+#endif
+#else
+ return lucene::util::pfor::decodeDelta1_128(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+#endif
}
size_t P4NZDEC(unsigned char* __restrict in, size_t n, uint32_t* __restrict out) {
- return g_p4nzdec(in, n, out);
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ return lucene::util::pfor::decodeZigzag256(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+#else
+ return DefaultDEC(in, nullptr, n, out);
+#endif
+#else
+ return lucene::util::pfor::decodeZigzag128(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+#endif
}
size_t P4ENC(uint32_t* __restrict in, size_t n, unsigned char* __restrict out) {
- return g_p4nd1enc(in, n, out);
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ return lucene::util::pfor::encodeDelta1_256(in, n, out);
+#else
+ return DefaultDENC(in, n, out);
+#endif
+#else
+ return lucene::util::pfor::encodeDelta1_128(in, n, out);
+#endif
}
size_t P4NZENC(uint32_t* __restrict in, size_t n, unsigned char* __restrict out) {
- return g_p4nzenc(in, n, out);
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ return lucene::util::pfor::encodeZigzag256(in, n, out);
+#else
+ return DefaultENC(in, n, out);
+#endif
+#else
+ return lucene::util::pfor::encodeZigzag128(in, n, out);
+#endif
}
void PforUtil::encodePos(IndexOutput* out, std::vector<uint32_t>& buffer) {
#ifdef __AVX2__
out->writeByte((char)CodeMode::kPfor256);
out->writeVInt(buffer.size());
- std::vector<uint8_t> compress(4 * buffer.size() + PFOR_BLOCK_SIZE);
- size_t compressSize = p4nzenc256v32(buffer.data(), buffer.size(), compress.data());
+ std::vector<uint8_t>& compress = scratch(lucene::util::pfor::encodedSizeBound(buffer.size()));
+ size_t compressSize =
+ lucene::util::pfor::encodeZigzag256(buffer.data(), buffer.size(), compress.data());
out->writeVInt(compressSize);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), compressSize);
#elif defined(__ARM_NEON)
out->writeByte((char)CodeMode::kPfor128);
out->writeVInt(buffer.size());
- std::vector<uint8_t> compress(4 * buffer.size() + PFOR_BLOCK_SIZE);
- size_t compressSize = p4nzenc32(buffer.data(), buffer.size(), compress.data());
+ std::vector<uint8_t>& compress = scratch(lucene::util::pfor::encodedSizeBound(buffer.size()));
+ size_t compressSize =
+ lucene::util::pfor::encodeZigzag128(buffer.data(), buffer.size(), compress.data());
out->writeVInt(compressSize);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), compressSize);
#else
@@ -170,25 +226,9 @@ uint32_t PforUtil::decodePos(IndexInput* in, std::vector<uint32_t>& buffer) {
if (mode == CodeMode::kPfor) {
_CLTHROWA(CL_ERR_CorruptIndex, "PFOR old version is not supported for decodePos");
} else if (mode == CodeMode::kPfor256) {
- uint32_t serializedSize = in->readVInt();
- std::vector<uint8_t> buf(serializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), serializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- p4nzdec256v32(buf.data(), size, buffer.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR256 is not supported on this platform");
-#endif
+ decodeSerialized(in, lucene::util::pfor::decodeZigzag256, size, buffer.data());
} else if (mode == CodeMode::kPfor128) {
- uint32_t serializedSize = in->readVInt();
- std::vector<uint8_t> buf(serializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), serializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- p4nzdec32(buf.data(), size, buffer.data());
-#elif defined(__ARM_NEON)
- p4nzdec32(buf.data(), size, buffer.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR128 is not supported on this platform");
-#endif
+ decodeSerialized(in, lucene::util::pfor::decodeZigzag128, size, buffer.data());
} else if (mode == CodeMode::kDefault) {
for (uint32_t i = 0; i < size; i++) {
buffer[i] = in->readVInt();
@@ -202,26 +242,32 @@ void PforUtil::pfor_encode(IndexOutput* out, std::vector<uint32_t>& docDeltaBuff
#ifdef __AVX2__
out->writeByte((char)CodeMode::kPfor256);
out->writeVInt(docDeltaBuffer.size());
- std::vector<uint8_t> compress(4 * docDeltaBuffer.size() + PFOR_BLOCK_SIZE);
+ std::vector<uint8_t>& compress =
+ scratch(lucene::util::pfor::encodedSizeBound(docDeltaBuffer.size()));
size_t size = 0;
- size = p4nd1enc256v32(docDeltaBuffer.data(), docDeltaBuffer.size(), compress.data());
+ size = lucene::util::pfor::encodeDelta1_256(docDeltaBuffer.data(), docDeltaBuffer.size(),
+ compress.data());
out->writeVInt(size);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), size);
if (has_prox) {
- size = p4nzenc256v32(freqBuffer.data(), freqBuffer.size(), compress.data());
+ size = lucene::util::pfor::encodeZigzag256(freqBuffer.data(), freqBuffer.size(),
+ compress.data());
out->writeVInt(size);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), size);
}
#elif defined(__ARM_NEON)
out->writeByte((char)CodeMode::kPfor128);
out->writeVInt(docDeltaBuffer.size());
- std::vector<uint8_t> compress(4 * docDeltaBuffer.size() + PFOR_BLOCK_SIZE);
+ std::vector<uint8_t>& compress =
+ scratch(lucene::util::pfor::encodedSizeBound(docDeltaBuffer.size()));
size_t size = 0;
- size = p4nd1enc32(docDeltaBuffer.data(), docDeltaBuffer.size(), compress.data());
+ size = lucene::util::pfor::encodeDelta1_128(docDeltaBuffer.data(), docDeltaBuffer.size(),
+ compress.data());
out->writeVInt(size);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), size);
if (has_prox) {
- size = p4nzenc32(freqBuffer.data(), freqBuffer.size(), compress.data());
+ size = lucene::util::pfor::encodeZigzag128(freqBuffer.data(), freqBuffer.size(),
+ compress.data());
out->writeVInt(size);
out->writeBytes(reinterpret_cast<const uint8_t*>(compress.data()), size);
}
@@ -255,65 +301,24 @@ uint32_t PforUtil::pfor_decode(IndexInput* in, std::vector<uint32_t>& docs,
std::vector<uint32_t>& freqs, bool has_prox, bool compatibleRead) {
char mode = in->readByte();
uint32_t arraySize = in->readVInt();
- // old version, need to separate read based on compatibleRead
+
+ DecodeFn docDecoder = nullptr;
+ DecodeFn freqDecoder = nullptr;
if (mode == (char)CodeMode::kPfor) {
- {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- // if compatibleRead is true, means we are reading old version arm64 index in x86_64 platform.
- if (compatibleRead) {
- p4nd1dec32(buf.data(), arraySize, docs.data());
- } else {
- p4nd1dec256v32(buf.data(), arraySize, docs.data());
- }
-#elif (defined(__ARM_NEON))
- // if compatibleRead is true, means we are reading old version x86_64 index in arm64 platform.
- if (compatibleRead) {
- p4nd1dec256scalarv32(buf.data(), arraySize, docs.data());
- } else {
- p4nd1dec32(buf.data(), arraySize, docs.data());
- }
-#elif (defined(__SSSE3__))
- // if compatibleRead is true, means we are reading old version x86_64 index in x86_64 which does not support avx2.
- if (compatibleRead) {
- p4nd1dec256scalarv32(buf.data(), arraySize, docs.data());
- } else {
- DefaultDDEC(buf.data(), arraySize, docs.data());
- }
-#else
- DefaultDDEC(buf.data(), arraySize, docs.data());
-#endif
- }
+ docDecoder = legacyDocDecoder(compatibleRead);
+ freqDecoder = legacyFreqDecoder(compatibleRead);
+ } else if (mode == (char)CodeMode::kPfor256) {
+ docDecoder = lucene::util::pfor::decodeDelta1_256;
+ freqDecoder = lucene::util::pfor::decodeZigzag256;
+ } else if (mode == (char)CodeMode::kPfor128) {
+ docDecoder = lucene::util::pfor::decodeDelta1_128;
+ freqDecoder = lucene::util::pfor::decodeZigzag128;
+ }
+
+ if (docDecoder != nullptr) {
+ decodeSerialized(in, docDecoder, arraySize, docs.data());
if (has_prox) {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- // if compatibleRead is true, means we are reading old version arm64 index in x86_64 platform.
- if (compatibleRead) {
- p4nzdec32(buf.data(), arraySize, freqs.data());
- } else {
- p4nzdec256v32(buf.data(), arraySize, freqs.data());
- }
-#elif (defined(__ARM_NEON))
- // if compatibleRead is true, means we are reading old version x86_64 index in arm64 platform.
- if (compatibleRead) {
- p4nzdec256scalarv32(buf.data(), arraySize, freqs.data());
- } else {
- p4nzdec32(buf.data(), arraySize, freqs.data());
- }
-#elif (defined(__SSSE3__))
- // if compatibleRead is true, means we are reading old version x86_64 index in x86_64 which does not support avx2.
- if (compatibleRead) {
- p4nzdec256scalarv32(buf.data(), arraySize, freqs.data());
- } else {
- DefaultDEC(buf.data(), arraySize, freqs.data());
- }
-#else
- DefaultDEC(buf.data(), arraySize, freqs.data());
-#endif
+ decodeSerialized(in, freqDecoder, arraySize, freqs.data());
}
} else if (mode == (char)CodeMode::kDefault) {
uint32_t docDelta = 0;
@@ -332,59 +337,6 @@ uint32_t PforUtil::pfor_decode(IndexInput* in, std::vector<uint32_t>& docs,
docs[i] = docDelta;
}
}
- } else if (mode == (char)CodeMode::kPfor256) {
- {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- // new version, read cross platform, x86_64 read arm64 index
- p4nd1dec256v32(buf.data(), arraySize, docs.data());
-#elif (defined(__SSSE3__) || defined(__ARM_NEON))
- p4nd1dec256scalarv32(buf.data(), arraySize, docs.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR256 is not supported on this platform");
-#endif
- }
- if (has_prox) {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- p4nzdec256v32(buf.data(), arraySize, freqs.data());
-#elif (defined(__SSSE3__) || defined(__ARM_NEON))
- // new version, read cross platform, arm64 read x86_64 index
- p4nzdec256scalarv32(buf.data(), arraySize, freqs.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR256 is not supported on this platform");
-#endif
- }
- } else if (mode == (char)CodeMode::kPfor128) {
- {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- // new version, read cross platform, x86_64 read arm64 index
- p4nd1dec32(buf.data(), arraySize, docs.data());
-#elif (defined(__SSSE3__) || defined(__ARM_NEON))
- p4nd1dec32(buf.data(), arraySize, docs.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR128 is not supported on this platform");
-#endif
- }
- if (has_prox) {
- uint32_t SerializedSize = in->readVInt();
- std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
- in->readBytes(buf.data(), SerializedSize);
-#if defined(USE_AVX2) && defined(__AVX2__)
- p4nzdec32(buf.data(), arraySize, freqs.data());
-#elif (defined(__SSSE3__) || defined(__ARM_NEON))
- p4nzdec32(buf.data(), arraySize, freqs.data());
-#else
- _CLTHROWA(CL_ERR_CorruptIndex, "PFOR128 is not supported on this platform");
-#endif
- }
}
return arraySize;
}
diff --git a/src/core/CLucene/util/PForCodec.cpp b/src/core/CLucene/util/PForCodec.cpp
new file mode 100644
index 00000000000..bbb1fae031a
--- /dev/null
+++ b/src/core/CLucene/util/PForCodec.cpp
@@ -0,0 +1,846 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "PForCodec.h"
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <utility>
+
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
+
+#include "CLucene/_ApiHeader.h"
+#include "CLucene/debug/error.h"
+
+static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "the stream layout assumes little endian");
+
+namespace lucene {
+namespace util {
+namespace pfor {
+
+namespace {
+
+constexpr uint32_t kModePlain = 0;
+constexpr uint32_t kModeList = 1;
+constexpr uint32_t kModeBitmap = 2;
+constexpr uint32_t kModeConst = 3;
+
+constexpr size_t kGroup = 32; // values per packing unit
+constexpr size_t kLanes = 8; // lanes of the vertical layout
+
+#if defined(__clang__)
+#define PFOR_UNROLL _Pragma("clang loop unroll(full)")
+#elif defined(__GNUC__)
+#define PFOR_UNROLL _Pragma("GCC unroll 32")
+#else
+#define PFOR_UNROLL
+#endif
+
+[[noreturn]] void corrupt(const char* what) {
+ _CLTHROWA(CL_ERR_CorruptIndex, what);
+}
+
+inline uint32_t bitLength(uint32_t x) {
+ return x == 0 ? 0 : 32 - static_cast<uint32_t>(__builtin_clz(x));
+}
+
+inline uint32_t load32(const uint8_t* p, size_t bytes) {
+ uint32_t v = 0;
+ memcpy(&v, p, bytes);
+ return v;
+}
+
+// ---------------------------------------------------------------------------------
+// Delta transforms
+// ---------------------------------------------------------------------------------
+
+// Eight 32-bit lanes; the compiler maps them to whatever vector registers the target has. The
+// functions passing them by value are internal, so the ABI note of GCC does not matter.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic ignored "-Wpsabi"
+#endif
+typedef uint32_t U32x8 __attribute__((vector_size(32)));
+typedef uint32_t U32x8Unaligned __attribute__((vector_size(32), aligned(4), may_alias));
+
+// Shuffles the lanes of two vectors, indices 0-7 pick from a and 8-15 from b. Clang and GCC 12+
+// know __builtin_shufflevector, older GCC only has __builtin_shuffle.
+#if defined(__clang__)
+#define PFOR_SHUFFLE(a, b, ...) __builtin_shufflevector(a, b, __VA_ARGS__)
+#else
+#define PFOR_SHUFFLE(a, b, ...) __builtin_shuffle(a, b, (U32x8) {__VA_ARGS__})
+#endif
+
+struct ZigzagDelta {
+ static uint32_t forward(uint32_t cur, uint32_t prev) {
+ uint32_t d = cur - prev;
+ return (d << 1) ^ (0u - (d >> 31));
+ }
+ // The signed difference a transformed value stands for.
+ static uint32_t toDelta(uint32_t t) { return (t >> 1) ^ (0u - (t & 1)); }
+ static U32x8 toDelta(U32x8 t) { return (t >> 1) ^ (0u - (t & 1u)); }
+};
+
+struct Delta1 {
+ static uint32_t forward(uint32_t cur, uint32_t prev) { return cur - prev - 1; }
+ static uint32_t toDelta(uint32_t t) { return t + 1; }
+ static U32x8 toDelta(U32x8 t) { return t + 1u; }
+};
+
+// ---------------------------------------------------------------------------------
+// Bit unpacking. A unit is 32 rows x L lanes of b-bit values stored in b*L words;
+// word j of lane l is at j*L+l and rows are packed LSB first. With L == 1 this is the
+// plain horizontal packing of 32 consecutive values, with L == 8 the vertical layout.
+// ---------------------------------------------------------------------------------
+
+#ifdef __AVX2__
+// Horizontal unpacking with AVX2. Eight consecutive b-bit values occupy exactly b bytes, so the
+// pattern of a group of eight is the same for the four groups of a unit. For b <= 25 every
+// value lies within four bytes: the bytes of the first four values are picked from 16 bytes
+// loaded at the start of the group, those of the other four from 16 bytes loaded at byte
+// 4*b/8, then shifted right by their bit offset.
+constexpr int kMaxShuffledWidth = 25;
+// The loads of the last group reach this many bytes past the 4*b bytes of a unit.
+constexpr size_t kUnpackOverread = 16;
+
+struct HorizontalTables {
+ alignas(32) uint8_t shuffle[kMaxShuffledWidth + 1][32];
+ alignas(32) uint32_t shift[kMaxShuffledWidth + 1][8];
+};
+
+constexpr HorizontalTables makeHorizontalTables() {
+ HorizontalTables t = {};
+ for (int b = 1; b <= kMaxShuffledWidth; ++b) {
+ for (int k = 0; k < 8; ++k) {
+ const int half = k / 4;
+ const int rel = k * b - (half ? 8 * ((4 * b) >> 3) : 0);
+ t.shift[b][k] = rel & 7;
+ for (int j = 0; j < 4; ++j) {
+ t.shuffle[b][half * 16 + (k % 4) * 4 + j] = (rel >> 3) + j;
+ }
+ }
+ }
+ return t;
+}
+
+constexpr HorizontalTables kHorizontalTables = makeHorizontalTables();
+#else
+constexpr size_t kUnpackOverread = 0;
+#endif
+
+#ifdef __AVX2__
+// Eight consecutive b-bit values, which occupy the b bytes at p (plus the overread of the loads).
+template <int B>
+inline void unpackGroup8(const uint8_t* p, uint32_t* out) {
+ if constexpr (B >= 1 && B <= kMaxShuffledWidth) {
+ const __m256i shuffle =
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kHorizontalTables.shuffle[B]));
+ const __m256i shift =
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kHorizontalTables.shift[B]));
+ const __m256i mask = _mm256_set1_epi32(static_cast<int>((1u << B) - 1));
+ const __m256i bytes = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))),
+ _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + ((4 * B) >> 3))), 1);
+ _mm256_storeu_si256(
+ reinterpret_cast<__m256i*>(out),
+ _mm256_and_si256(_mm256_srlv_epi32(_mm256_shuffle_epi8(bytes, shuffle), shift),
+ mask));
+ }
+}
+
+using Group8Fn = void (*)(const uint8_t*, uint32_t*);
+
+template <size_t... B>
+constexpr std::array<Group8Fn, 33> makeGroup8Table(std::index_sequence<B...>) {
+ return {{(B >= 1 && B <= kMaxShuffledWidth ? &unpackGroup8<static_cast<int>(B)> : nullptr)...}};
+}
+
+constexpr std::array<Group8Fn, 33> kUnpackGroup8 = makeGroup8Table(std::make_index_sequence<33> {});
+#else
+using Group8Fn = void (*)(const uint8_t*, uint32_t*);
+constexpr std::array<Group8Fn, 33> kUnpackGroup8 = {};
+#endif
+
+template <int B, int L>
+void unpackUnit(const uint8_t* in, uint32_t* out) {
+ if constexpr (B == 0) {
+ memset(out, 0, kGroup * L * sizeof(uint32_t));
+#ifdef __AVX2__
+ } else if constexpr (L == 1 && B <= kMaxShuffledWidth) {
+ for (int g = 0; g < 4; ++g) {
+ unpackGroup8<B>(in + g * B, out + 8 * g);
+ }
+#endif
+ } else {
+ uint32_t w[B * L];
+ memcpy(w, in, sizeof(w));
+ constexpr uint32_t mask = B == 32 ? ~0u : (1u << (B % 32)) - 1;
+ PFOR_UNROLL
+ for (int i = 0; i < 32; ++i) {
+ const int bit = i * B;
+ const int j = bit >> 5;
+ const int s = bit & 31;
+ for (int l = 0; l < L; ++l) {
+ uint32_t v = w[j * L + l] >> s;
+ if (s + B > 32) {
+ v |= w[(j + 1) * L + l] << (32 - s);
+ }
+ out[i * L + l] = v & mask;
+ }
+ }
+ }
+}
+
+using UnpackFn = void (*)(const uint8_t*, uint32_t*);
+
+template <int L, size_t... B>
+constexpr std::array<UnpackFn, 33> makeUnpackTable(std::index_sequence<B...>) {
+ return {{&unpackUnit<static_cast<int>(B), L>...}};
+}
+
+constexpr std::array<UnpackFn, 33> kUnpackHorizontal =
+ makeUnpackTable<1>(std::make_index_sequence<33> {});
+constexpr std::array<UnpackFn, 33> kUnpackVertical =
+ makeUnpackTable<kLanes>(std::make_index_sequence<33> {});
+
+// The inverse of unpackUnit: packs 32 rows x L lanes (values at i*L+l) into B*L words.
+template <int B, int L>
+void packUnit(const uint32_t* v, uint32_t* words) {
+ if constexpr (B != 0) {
+ uint32_t w[B * L] = {};
+ constexpr uint32_t mask = B == 32 ? ~0u : (1u << (B % 32)) - 1;
+ PFOR_UNROLL
+ for (int i = 0; i < 32; ++i) {
+ const int bit = i * B;
+ const int j = bit >> 5;
+ const int s = bit & 31;
+ for (int l = 0; l < L; ++l) {
+ const uint32_t x = v[i * L + l] & mask;
+ w[j * L + l] |= x << s;
+ if (s + B > 32) {
+ w[(j + 1) * L + l] |= x >> (32 - s);
+ }
+ }
+ }
+ memcpy(words, w, sizeof(w));
+ }
+}
+
+using PackFn = void (*)(const uint32_t*, uint32_t*);
+
+template <int L, size_t... B>
+constexpr std::array<PackFn, 33> makePackTable(std::index_sequence<B...>) {
+ return {{&packUnit<static_cast<int>(B), L>...}};
+}
+
+constexpr std::array<PackFn, 33> kPackHorizontal =
+ makePackTable<1>(std::make_index_sequence<33> {});
+constexpr std::array<PackFn, 33> kPackVertical =
+ makePackTable<kLanes>(std::make_index_sequence<33> {});
+
+inline size_t packedSize(size_t k, uint32_t b, bool vertical) {
+ return vertical ? kGroup * b : (k * b + 7) / 8;
+}
+
+// Writes k values of width b, returns the number of bytes written.
+size_t packValues(const uint32_t* t, size_t k, uint32_t b, bool vertical, uint8_t* out) {
+ if (b == 0) {
+ return 0;
+ }
+ if (vertical) {
+ uint32_t words[kLanes * 32];
+ kPackVertical[b](t, words);
+ memcpy(out, words, kGroup * b);
+ return kGroup * b;
+ }
+ uint8_t* o = out;
+ for (size_t g = 0; g < k; g += kGroup) {
+ const size_t n = std::min(kGroup, k - g);
+ uint32_t words[32];
+ if (n == kGroup) {
+ kPackHorizontal[b](t + g, words);
+ } else {
+ uint32_t buf[kGroup] = {};
+ memcpy(buf, t + g, n * sizeof(uint32_t));
+ kPackHorizontal[b](buf, words);
+ }
+ const size_t bytes = n == kGroup ? 4 * b : (n * b + 7) / 8;
+ memcpy(o, words, bytes);
+ o += bytes;
+ }
+ return o - out;
+}
+
+// ---------------------------------------------------------------------------------
+// Variable length integers
+// ---------------------------------------------------------------------------------
+
+constexpr uint32_t kVarint1 = 177;
+constexpr uint32_t kVarint2 = kVarint1 + (64u << 8);
+constexpr uint32_t kVarint3 = kVarint2 + (8u << 16);
+
+uint8_t* putFirst(uint8_t* o, uint32_t x) {
+ if (x < (1u << 7)) {
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < (1u << 14)) {
+ *o++ = static_cast<uint8_t>(0x80 | (x >> 8));
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < (1u << 21)) {
+ *o++ = static_cast<uint8_t>(0xc0 | (x >> 16));
+ memcpy(o, &x, 2);
+ o += 2;
+ } else if (x < (1u << 28)) {
+ *o++ = static_cast<uint8_t>(0xe0 | (x >> 24));
+ *o++ = static_cast<uint8_t>(x >> 16);
+ *o++ = static_cast<uint8_t>(x >> 8);
+ *o++ = static_cast<uint8_t>(x);
+ } else {
+ *o++ = 0xf0;
+ memcpy(o, &x, 4);
+ o += 4;
+ }
+ return o;
+}
+
+// The hi parts of a list block are either that many variable length integers or, if that
+// is smaller, the marker kRawList followed by plain 32-bit values.
+constexpr uint32_t kRawList = 0xff;
+constexpr uint32_t kVarint4 = 1u << 24;
+
+uint8_t* putVarint(uint8_t* o, uint32_t x) {
+ if (x < kVarint1) {
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < kVarint2) {
+ x -= kVarint1;
+ *o++ = static_cast<uint8_t>(0xb1 + (x >> 8));
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < kVarint3) {
+ x -= kVarint2;
+ *o++ = static_cast<uint8_t>(0xf1 + (x >> 16));
+ memcpy(o, &x, 2);
+ o += 2;
+ } else if (x < kVarint4) {
+ *o++ = 0xf9;
+ memcpy(o, &x, 3);
+ o += 3;
+ } else {
+ *o++ = 0xfa;
+ memcpy(o, &x, 4);
+ o += 4;
+ }
+ return o;
+}
+
+// Number of t[i] >= threshold. Written as a plain count so that it is vectorized.
+inline size_t countAtLeast(const uint32_t* t, size_t k, uint64_t threshold) {
+ if (threshold > UINT32_MAX) {
+ return 0;
+ }
+ const uint32_t limit = static_cast<uint32_t>(threshold);
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ n += t[i] >= limit;
+ }
+ return n;
+}
+
+// Bytes of the hi parts (t >> b) of the `exceptions` values wider than b, written as variable
+// length integers; maxBits is the width of the widest value.
+inline size_t hiVarintBytes(const uint32_t* t, size_t k, uint32_t b, uint32_t maxBits,
+ size_t exceptions) {
+ if (maxBits - b <= 7) {
+ return exceptions; // every hi part is below 128, one byte each
+ }
+ return exceptions + countAtLeast(t, k, static_cast<uint64_t>(kVarint1) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint2) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint3) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint4) << b);
+}
+
+// Size of the hi parts section of a list block with n exceptions; sets `raw` if the raw form
+// is the smaller.
+inline size_t hiListSize(size_t varints, size_t n, bool& raw) {
+ const size_t rawSize = 1 + 4 * n;
+ raw = rawSize < varints;
+ return raw ? rawSize : varints;
+}
+
+// ---------------------------------------------------------------------------------
+// Block encoder
+// ---------------------------------------------------------------------------------
+
+// Encodes the k deltas t[0..k) as one block and returns the number of bytes written.
+size_t encodeBlock(const uint32_t* t, size_t k, bool vertical, uint8_t* out) {
+ uint8_t* o = out;
+
+ if (std::all_of(t, t + k, [&](uint32_t x) { return x == t[0]; })) {
+ if (t[0] == 0) {
+ *o++ = kModePlain << 6;
+ } else {
+ const uint32_t b = bitLength(t[0]);
+ *o++ = static_cast<uint8_t>(kModeConst << 6 | b);
+ memcpy(o, &t[0], (b + 7) / 8);
+ o += (b + 7) / 8;
+ }
+ return o - out;
+ }
+
+ uint32_t any = 0;
+ for (size_t i = 0; i < k; ++i) {
+ any |= t[i];
+ }
+ const uint32_t maxBits = bitLength(any);
+
+ // Choose the packed width and the way the values wider than it are patched.
+ struct Plan {
+ size_t cost;
+ uint32_t mode;
+ uint32_t b;
+ };
+ Plan best = {1 + packedSize(k, maxBits, vertical), kModePlain, maxBits};
+ size_t fewest = 0; // the exceptions at the previous (wider) b; there are at least as many now
+ for (int64_t b = static_cast<int64_t>(maxBits) - 1; b >= 0; --b) {
+ const uint32_t bx = maxBits - static_cast<uint32_t>(b);
+ const size_t packed = packedSize(k, static_cast<uint32_t>(b), vertical);
+
+ // Neither the list (2 bytes + a byte per position and per hi part) nor the bitmap can be
+ // smaller than this, so this width cannot win and its exceptions need not be counted.
+ if (2 + packed + std::min(2 * fewest, (k + 7) / 8 + (fewest * bx + 7) / 8) >= best.cost) {
+ continue;
+ }
+ const size_t exceptions = countAtLeast(t, k, uint64_t(1) << b);
+ fewest = exceptions;
+
+ const size_t bitmapCost = 2 + (k + 7) / 8 + (exceptions * bx + 7) / 8 + packed;
+ if (bitmapCost < best.cost) {
+ best = {bitmapCost, kModeBitmap, static_cast<uint32_t>(b)};
+ }
+ const size_t listBase = 2 + packed + exceptions;
+ if (exceptions <= 255 && listBase + exceptions < best.cost) {
+ bool raw;
+ const size_t cost = listBase + hiListSize(hiVarintBytes(t, k, static_cast<uint32_t>(b),
+ maxBits, exceptions),
+ exceptions, raw);
+ if (cost < best.cost) {
+ best = {cost, kModeList, static_cast<uint32_t>(b)};
+ }
+ }
+ }
+
+ const uint32_t b = best.b;
+ *o++ = static_cast<uint8_t>(best.mode << 6 | b);
+ if (best.mode == kModePlain) {
+ o += packValues(t, k, b, vertical, o);
+ } else if (best.mode == kModeList) {
+ uint8_t* count = o++;
+ o += packValues(t, k, b, vertical, o);
+ uint8_t positions[256];
+ uint32_t hi[256];
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ if (t[i] >> b != 0) {
+ hi[n] = t[i] >> b;
+ positions[n++] = static_cast<uint8_t>(i);
+ }
+ }
+ bool raw;
+ hiListSize(hiVarintBytes(t, k, b, maxBits, n), n, raw);
+ if (raw) {
+ *o++ = kRawList;
+ memcpy(o, hi, 4 * n);
+ o += 4 * n;
+ } else {
+ for (size_t i = 0; i < n; ++i) {
+ o = putVarint(o, hi[i]);
+ }
+ }
+ *count = static_cast<uint8_t>(n);
+ memcpy(o, positions, n);
+ o += n;
+ } else {
+ *o++ = static_cast<uint8_t>(maxBits - b);
+ uint8_t* bitmap = o;
+ memset(bitmap, 0, (k + 7) / 8);
+ o += (k + 7) / 8;
+ uint32_t hi[256];
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ if (t[i] >> b != 0) {
+ bitmap[i >> 3] |= static_cast<uint8_t>(1u << (i & 7));
+ hi[n++] = t[i] >> b;
+ }
+ }
+ o += packValues(hi, n, maxBits - b, false, o);
+ o += packValues(t, k, b, vertical, o);
+ }
+ return o - out;
+}
+
+// ---------------------------------------------------------------------------------
+// Block decoder
+// ---------------------------------------------------------------------------------
+
+struct Reader {
+ const uint8_t* p;
+ const uint8_t* end;
+
+ void need(size_t n) const {
+ if (static_cast<size_t>(end - p) < n) {
+ corrupt("PFOR: truncated block");
+ }
+ }
+ uint8_t u8() {
+ need(1);
+ return *p++;
+ }
+};
+
+uint32_t getFirst(Reader& r) {
+ const uint32_t b0 = r.u8();
+ if (b0 < 0x80) {
+ return b0;
+ }
+ if (b0 < 0xc0) {
+ return ((b0 & 0x3f) << 8) | r.u8();
+ }
+ if (b0 < 0xe0) {
+ r.need(2);
+ const uint32_t v = ((b0 & 0x1f) << 16) | load32(r.p, 2);
+ r.p += 2;
+ return v;
+ }
+ if (b0 < 0xf0) {
+ r.need(3);
+ const uint32_t v = ((b0 & 0x0f) << 24) | (static_cast<uint32_t>(r.p[0]) << 16) |
+ (static_cast<uint32_t>(r.p[1]) << 8) | r.p[2];
+ r.p += 3;
+ return v;
+ }
+ if (b0 != 0xf0) {
+ corrupt("PFOR: bad first value");
+ }
+ r.need(4);
+ const uint32_t v = load32(r.p, 4);
+ r.p += 4;
+ return v;
+}
+
+uint32_t getVarint(Reader& r) {
+ const uint32_t b0 = r.u8();
+ if (b0 < 0xb1) {
+ return b0;
+ }
+ if (b0 <= 0xf0) {
+ return kVarint1 + (((b0 - 0xb1) << 8) | r.u8());
+ }
+ if (b0 <= 0xf8) {
+ r.need(2);
+ const uint32_t v = kVarint2 + (((b0 - 0xf1) << 16) | load32(r.p, 2));
+ r.p += 2;
+ return v;
+ }
+ if (b0 == 0xf9) {
+ r.need(3);
+ const uint32_t v = load32(r.p, 3);
+ r.p += 3;
+ return v;
+ }
+ if (b0 == 0xfa) {
+ r.need(4);
+ const uint32_t v = load32(r.p, 4);
+ r.p += 4;
+ return v;
+ }
+ corrupt("PFOR: bad exception value");
+}
+
+// Reads k packed values of width b.
+void unpackValues(Reader& r, size_t k, uint32_t b, bool vertical, uint32_t* t) {
+ const size_t bytes = packedSize(k, b, vertical);
+ r.need(bytes);
+ if (vertical) {
+ kUnpackVertical[b](r.p, t);
+ } else {
+ // Units are read with kUnpackOverread bytes more than they have, which kDecodePadding
+ // allows past the end.
+ static_assert(kUnpackOverread <= kDecodePadding, "see PForCodec.h");
+ static_assert(8 <= kDecodePadding, "see PForCodec.h");
+ size_t g = 0;
+ const uint8_t* p = r.p;
+ for (; g + kGroup <= k; g += kGroup, p += 4 * b) {
+ kUnpackHorizontal[b](p, t + g);
+ }
+ // fewer than 32 values left: groups of 8 as far as possible, then each value comes from
+ // the 8 bytes at its first byte
+ size_t i = 0;
+ if (const Group8Fn group8 = kUnpackGroup8[b]) {
+ for (; g + i + 8 <= k; i += 8) {
+ group8(p + (i / 8) * b, t + g + i);
+ }
+ }
+ const uint32_t mask = b >= 32 ? ~0u : (1u << b) - 1;
+ for (; g + i < k; ++i) {
+ uint64_t word;
+ memcpy(&word, p + ((i * b) >> 3), sizeof(word));
+ t[g + i] = static_cast<uint32_t>(word >> ((i * b) & 7)) & mask;
+ }
+ }
+ r.p += bytes;
+}
+
+#ifdef __AVX2__
+// For a byte of the exception bitmap, the index into the (still packed) hi parts of the value of
+// each lane: the number of exceptions before that lane.
+struct ExpandTable {
+ alignas(32) uint32_t index[256][8];
+ alignas(32) uint32_t mask[256][8]; // all ones for the lanes whose bit is set
+};
+
+constexpr ExpandTable makeExpandTable() {
+ ExpandTable t = {};
+ for (int bits = 0; bits < 256; ++bits) {
+ uint32_t before = 0;
+ for (int lane = 0; lane < 8; ++lane) {
+ t.index[bits][lane] = before;
+ t.mask[bits][lane] = ((bits >> lane) & 1) ? ~0u : 0u;
+ before += (bits >> lane) & 1;
+ }
+ }
+ return t;
+}
+
+constexpr ExpandTable kExpandTable = makeExpandTable();
+#endif
+
+// t[i] |= hi[n] << b for the n-th set bit i of the bitmap. `hi` must have 8 readable entries
+// after the last exception.
+inline void applyBitmapExceptions(uint32_t* t, const uint8_t* bitmap, size_t k, const uint32_t* hi,
+ uint32_t b) {
+ size_t n = 0;
+ size_t byte = 0;
+#ifdef __AVX2__
+ // where the hi parts of each byte start; computed first so that the bytes below do not wait
+ // for each other
+ uint32_t before[33];
+ before[0] = 0;
+ for (size_t j = 0; j < k / 8; ++j) {
+ before[j + 1] = before[j] + __builtin_popcount(bitmap[j]);
+ }
+ const __m128i shift = _mm_cvtsi32_si128(static_cast<int>(b));
+ for (; byte < k / 8; ++byte) {
+ const uint32_t bits = bitmap[byte];
+ const __m256i spread = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(hi + before[byte])),
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kExpandTable.index[bits])));
+ __m256i* dst = reinterpret_cast<__m256i*>(t + 8 * byte);
+ _mm256_storeu_si256(
+ dst, _mm256_or_si256(
+ _mm256_loadu_si256(dst),
+ _mm256_and_si256(_mm256_sll_epi32(spread, shift),
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(
+ kExpandTable.mask[bits])))));
+ }
+ n = before[k / 8];
+#endif
+ for (; byte < (k + 7) / 8; ++byte) {
+ for (uint32_t bits = bitmap[byte]; bits != 0; bits &= bits - 1) {
+ const size_t i = byte * 8 + __builtin_ctz(bits);
+ if (i >= k) {
+ corrupt("PFOR: bad exception position");
+ }
+ t[i] |= hi[n++] << b;
+ }
+ }
+}
+
+// Decodes the next block of k deltas into t.
+void decodeBlock(Reader& r, size_t k, bool vertical, uint32_t* t) {
+ const uint32_t header = r.u8();
+ const uint32_t mode = header >> 6;
+ const uint32_t b = header & 0x3f;
+ if (b > 32) {
+ corrupt("PFOR: bad bit width");
+ }
+
+ if (mode == kModePlain) {
+ unpackValues(r, k, b, vertical, t);
+ } else if (mode == kModeConst) {
+ const size_t bytes = (b + 7) / 8;
+ r.need(bytes);
+ static_assert(4 <= kDecodePadding, "see PForCodec.h");
+ // always load 4 bytes, which is cheaper than a load of a variable size
+ const uint32_t v = load32(r.p, 4) & (bytes >= 4 ? ~0u : (1u << (8 * bytes)) - 1);
+ r.p += bytes;
+ std::fill(t, t + k, v);
+ } else if (mode == kModeList) {
+ if (b >= 32) {
+ corrupt("PFOR: bad bit width");
+ }
+ const size_t count = r.u8();
+ if (count > k) {
+ corrupt("PFOR: bad exception count");
+ }
+ unpackValues(r, k, b, vertical, t);
+ uint32_t hi[256];
+ if (count > 0) {
+ r.need(1);
+ if (*r.p == kRawList) {
+ ++r.p;
+ r.need(4 * count);
+ memcpy(hi, r.p, 4 * count);
+ r.p += 4 * count;
+ } else {
+ for (size_t i = 0; i < count; ++i) {
+ hi[i] = getVarint(r);
+ }
+ }
+ }
+ r.need(count);
+ for (size_t i = 0; i < count; ++i) {
+ const size_t pos = r.p[i];
+ if (pos >= k) {
+ corrupt("PFOR: bad exception position");
+ }
+ t[pos] |= hi[i] << b;
+ }
+ r.p += count;
+ } else {
+ const uint32_t bx = r.u8();
+ if (b >= 32 || bx == 0 || b + bx > 32) {
+ corrupt("PFOR: bad bit width");
+ }
+ const size_t bitmapBytes = (k + 7) / 8;
+ r.need(bitmapBytes);
+ const uint8_t* bitmap = r.p;
+ r.p += bitmapBytes;
+ size_t count = 0;
+ for (size_t i = 0; i < bitmapBytes; ++i) {
+ count += __builtin_popcount(bitmap[i]);
+ }
+ uint32_t hi[256 + 8];
+ unpackValues(r, count, bx, false, hi);
+ memset(hi + count, 0, 8 * sizeof(uint32_t)); // the vector code reads whole registers
+ unpackValues(r, k, b, vertical, t);
+ applyBitmapExceptions(t, bitmap, k, hi, b);
+ }
+}
+
+// ---------------------------------------------------------------------------------
+// Whole stream
+// ---------------------------------------------------------------------------------
+
+template <class Transform, size_t kBlock, bool kVertical>
+size_t encodeImpl(const uint32_t* in, size_t n, uint8_t* out) {
+ if (n == 0) {
+ return 0;
+ }
+ uint8_t* o = putFirst(out, in[0]);
+ uint32_t t[kBlock];
+ for (size_t i = 1; i < n;) {
+ const size_t k = std::min(kBlock, n - i);
+ for (size_t j = 0; j < k; ++j) {
+ t[j] = Transform::forward(in[i + j], in[i + j - 1]);
+ }
+ o += encodeBlock(t, k, kVertical && k == kBlock, o);
+ i += k;
+ }
+ return o - out;
+}
+
+// Inclusive prefix sum within the 8 lanes.
+inline U32x8 scan8(U32x8 x) {
+ const U32x8 zero = {};
+ x += PFOR_SHUFFLE(x, zero, 8, 0, 1, 2, 3, 4, 5, 6);
+ x += PFOR_SHUFFLE(x, zero, 8, 8, 0, 1, 2, 3, 4, 5);
+ x += PFOR_SHUFFLE(x, zero, 8, 8, 8, 8, 0, 1, 2, 3);
+ return x;
+}
+
+// Turns the k transformed values t into out[i] = cur + delta(t[0]) + ... + delta(t[i]) and
+// returns the last value. Eight values are scanned at a time; the scan of a group does not
+// depend on the running total, which only costs one add per group.
+template <class Transform>
+inline uint32_t accumulate(uint32_t cur, const uint32_t* t, size_t k, uint32_t* out) {
+ size_t i = 0;
+ if (k >= 8) {
+ U32x8 total = {cur, cur, cur, cur, cur, cur, cur, cur};
+ for (; i + 8 <= k; i += 8) {
+ U32x8 x = scan8(Transform::toDelta(*reinterpret_cast<const U32x8Unaligned*>(t + i)));
+ *reinterpret_cast<U32x8Unaligned*>(out + i) = x + total;
+ total += PFOR_SHUFFLE(x, x, 7, 7, 7, 7, 7, 7, 7, 7);
+ }
+ cur = total[0];
+ }
+ for (; i < k; ++i) {
+ cur += Transform::toDelta(t[i]);
+ out[i] = cur;
+ }
+ return cur;
+}
+
+// Everything it calls is inlined: the arguments are few and the reader state stays in registers.
+template <class Transform, size_t kBlock, bool kVertical>
+__attribute__((flatten)) size_t decodeImpl(const uint8_t* in, const uint8_t* end, size_t n,
+ uint32_t* out) {
+ if (n == 0) {
+ return 0;
+ }
+ Reader r = {in, end};
+ uint32_t cur = getFirst(r);
+ out[0] = cur;
+ alignas(32) uint32_t t[kBlock];
+ for (size_t i = 1; i < n;) {
+ const size_t k = std::min(kBlock, n - i);
+ decodeBlock(r, k, kVertical && k == kBlock, t);
+ cur = accumulate<Transform>(cur, t, k, out + i);
+ i += k;
+ }
+ return r.p - in;
+}
+
+} // namespace
+
+size_t encodeZigzag128(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<ZigzagDelta, 128, false>(in, n, out);
+}
+size_t encodeZigzag256(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<ZigzagDelta, 256, true>(in, n, out);
+}
+size_t encodeDelta1_128(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<Delta1, 128, false>(in, n, out);
+}
+size_t encodeDelta1_256(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<Delta1, 256, true>(in, n, out);
+}
+
+size_t decodeZigzag128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<ZigzagDelta, 128, false>(in, end, n, out);
+}
+size_t decodeZigzag256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<ZigzagDelta, 256, true>(in, end, n, out);
+}
+size_t decodeDelta1_128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<Delta1, 128, false>(in, end, n, out);
+}
+size_t decodeDelta1_256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<Delta1, 256, true>(in, end, n, out);
+}
+
+} // namespace pfor
+} // namespace util
+} // namespace lucene
diff --git a/src/core/CLucene/util/PForCodec.h b/src/core/CLucene/util/PForCodec.h
new file mode 100644
index 00000000000..e705b1f5268
--- /dev/null
+++ b/src/core/CLucene/util/PForCodec.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Patched frame-of-reference codec for uint32 arrays, used by the inverted index
+// postings. This is an independent implementation of the on-disk block format that
+// has been written into existing indexes (CodeMode kPfor / kPfor128 / kPfor256), so
+// that those indexes stay readable and newly written ones stay readable by old
+// versions.
+//
+// Stream layout (all multi-byte fields little endian unless noted):
+//
+// stream := first-value block*
+// first-value : the array's first element, 1..5 bytes
+// x < 2^7 : x
+// x < 2^14 : 0x80|x>>8, x
+// x < 2^21 : 0xc0|x>>16, x[16]
+// x < 2^28 : 0xe0|x>>24, x>>16, x>>8, x (big endian tail)
+// otherwise : 0xf0, x[32]
+//
+// The remaining n-1 elements are turned into n-1 "deltas" t[i] (see Transform below)
+// and cut into blocks of kBlock deltas; the last block holds the remainder. Every
+// block starts with a header byte h = mode<<6 | b, where b (0..32) is the bit width
+// of the packed part:
+//
+// mode 0 (plain) : packed(b)
+// mode 1 (list) : count, packed(b) of the low b bits, his, count x pos
+// his holds the hi parts (x >> b) of the exceptions: count variable
+// length integers, or the byte 0xff followed by count plain 32-bit
+// values. pos is the index in the block (1 byte).
+// mode 2 (bitmap) : bx, bitmap[ceil(k/8)], packed(bx) of the hi parts (x >> b) of the
+// flagged values in index order, packed(b) of the low b bits
+// mode 3 (const) : value[ceil(b/8)], all k deltas equal to value
+//
+// packed(w) of k values is LSB-first bit packing. It is "horizontal" (value i at bit
+// i*w) except for a full 256 value block of the 256 layout, which is "vertical":
+// value i lives in lane i%8, row i/8; each lane packs its 32 rows into w 32-bit words
+// and word j of lane l is stored at word index j*8+l.
+//
+// Variable length integer for hi parts:
+// x < 177 : x
+// x < 177+2^14 : 0xb1+(x-177)>>8, (x-177)
+// x < 16561+2^19 : 0xf1+(x-16561)>>16, (x-16561)[16]
+// x < 2^24 : 0xf9, x[24]
+// otherwise : 0xfa, x[32]
+namespace lucene {
+namespace util {
+namespace pfor {
+
+// Upper bound of the encoded size of n values, for sizing output buffers.
+constexpr size_t encodedSizeBound(size_t n) {
+ return 8 + n * 4 + n / 128 + 1;
+}
+
+// Delta = zigzag(in[i] - in[i-1]).
+size_t encodeZigzag128(const uint32_t* in, size_t n, uint8_t* out);
+size_t encodeZigzag256(const uint32_t* in, size_t n, uint8_t* out);
+// Delta = in[i] - in[i-1] - 1 (strictly increasing sequences).
+size_t encodeDelta1_128(const uint32_t* in, size_t n, uint8_t* out);
+size_t encodeDelta1_256(const uint32_t* in, size_t n, uint8_t* out);
+
+// The decoders may load up to kDecodePadding bytes past `end` (the loaded values are never
+// used), so the buffer must be readable that far.
+constexpr size_t kDecodePadding = 16;
+
+// Decoders write exactly n values to out and return the number of input bytes consumed.
+// Nothing at or after `end` is interpreted; a malformed or truncated stream throws
+// CL_ERR_CorruptIndex.
+size_t decodeZigzag128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeZigzag256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeDelta1_128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeDelta1_256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+
+} // namespace pfor
+} // namespace util
+} // namespace lucene
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 0a19a2d278f..83757d147fb 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -56,6 +56,7 @@ SET(clucene_core_Files
./CLucene/util/NumericUtils.cpp
./CLucene/util/stringUtil.cpp
./CLucene/util/PFORUtil.cpp
+ ./CLucene/util/PForCodec.cpp
./CLucene/queryParser/FastCharStream.cpp
./CLucene/queryParser/MultiFieldQueryParser.cpp
./CLucene/queryParser/QueryParser.cpp
@@ -213,7 +214,6 @@ file(GLOB_RECURSE HEADERS ${clucene-core_SOURCE_DIR}/*.h )
#create the libraries
INCLUDE_DIRECTORIES(${clucene_SOURCE_DIR}/src/core)
-INCLUDE_DIRECTORIES(${clucene_SOURCE_DIR}/src/ext/for)
IF (BUILD_SHARED_LIBRARIES)
add_library(clucene-core SHARED
diff --git a/src/demo/CMakeLists.txt b/src/demo/CMakeLists.txt
index 171e554a9f3..5b054832f16 100644
--- a/src/demo/CMakeLists.txt
+++ b/src/demo/CMakeLists.txt
@@ -37,12 +37,10 @@ ADD_EXECUTABLE(cl_demo_index EXCLUDE_FROM_ALL
#./TestAnalyzer.cpp
#${demo_HEADERS}
#)
-#add_library(ic STATIC IMPORTED)
-#set_target_properties(ic PROPERTIES IMPORTED_LOCATION "${clucene-ext_SOURCE_DIR}/for/libic.a")
-TARGET_LINK_LIBRARIES(cl_demo clucene-core-static clucene-shared-static ic ${EXTRA_LIBS})
-TARGET_LINK_LIBRARIES(cl_demo_index clucene-core-static clucene-shared-static ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(cl_demo clucene-core-static clucene-shared-static ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(cl_demo_index clucene-core-static clucene-shared-static ${EXTRA_LIBS})
ADD_EXECUTABLE(allIndex EXCLUDE_FROM_ALL
./allIndex.cpp
@@ -51,7 +49,7 @@ ADD_EXECUTABLE(allIndex EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(allIndex clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(allIndex clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(abnormal EXCLUDE_FROM_ALL
./abnormal.cpp
@@ -60,7 +58,7 @@ ADD_EXECUTABLE(abnormal EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(abnormal clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(abnormal clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(search EXCLUDE_FROM_ALL
./search.cpp
@@ -68,7 +66,7 @@ ADD_EXECUTABLE(search EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(search clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(search clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(strSearch EXCLUDE_FROM_ALL
./strSearch.cpp
@@ -77,7 +75,7 @@ ADD_EXECUTABLE(strSearch EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(strSearch clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(strSearch clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(pointSearch4 EXCLUDE_FROM_ALL
./pointSearch4.cpp
@@ -86,7 +84,7 @@ ADD_EXECUTABLE(pointSearch4 EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(pointSearch4 clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(pointSearch4 clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(pointSearch8 EXCLUDE_FROM_ALL
./pointSearch8.cpp
@@ -95,7 +93,7 @@ ADD_EXECUTABLE(pointSearch8 EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(pointSearch8 clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(pointSearch8 clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(demo EXCLUDE_FROM_ALL
./demo.cpp
@@ -104,4 +102,4 @@ ADD_EXECUTABLE(demo EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(demo clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
\ No newline at end of file
+TARGET_LINK_LIBRARIES(demo clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
\ No newline at end of file
diff --git a/src/ext/for/CMakeLists.txt b/src/ext/for/CMakeLists.txt
deleted file mode 100644
index 24384297462..00000000000
--- a/src/ext/for/CMakeLists.txt
+++ /dev/null
@@ -1,107 +0,0 @@
-cmake_minimum_required(VERSION 3.10)
-project(powturbo)
-
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_C_STANDARD 99)
-
-if(NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Release)
-endif()
-
-set(DEBUG "-DNDEBUG -g -O3")
-set(OPT "-w -Wall -fstrict-aliasing -falign-loops -Wno-int-conversion")
-
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
- set(SSE "-msse4.2")
- set(AVX2 "-mavx2")
- set(CMAKE_C_FLAGS ${SSE})
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
- set(SSE "-march=armv8-a")
- set(CMAKE_C_FLAGS "-march=armv8-a")
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
- set(SSE "-D__SSSE3__")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power9 -mtune=power9")
-endif()
-
-if(FLOAT16)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_FLOAT16")
-endif()
-
-if(STATIC)
- set(CMAKE_EXE_LINKER_FLAGS "-static")
-endif()
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${DEBUG} ${OPT}")
-
-separate_arguments(avx2_c_flags_list UNIX_COMMAND "${DEBUG} ${OPT}")
-separate_arguments(c_flags_list UNIX_COMMAND "${CMAKE_C_FLAGS}")
-
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-
-set(SRC_FILES
- ${CMAKE_CURRENT_SOURCE_DIR}/bitpack.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bitunpack.c
- ${CMAKE_CURRENT_SOURCE_DIR}/vp4c.c
- ${CMAKE_CURRENT_SOURCE_DIR}/vp4d.c
- ${CMAKE_CURRENT_SOURCE_DIR}/transpose.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bitutil.c
-)
-
-add_library(ic STATIC
- v8.c
- vint.c
- trlec.c
- trled.c
- vsimple.c
- bitutil.c
- bitpack.c
- bitunpack.c
- vp4c.c
- vp4d.c
- transpose.c
-)
-
-foreach(SRC_FILE ${SRC_FILES})
- get_filename_component(SRC_NAME ${SRC_FILE} NAME_WE)
- set(SSE_OUTPUT ${SRC_NAME}_sse.o)
- set(AVX2_OUTPUT ${SRC_NAME}_avx2.o)
-
- add_custom_command(
- OUTPUT ${SSE_OUTPUT}
- COMMAND ${CMAKE_C_COMPILER} ${c_flags_list} -c -o ${SSE_OUTPUT} ${SRC_FILE}
- DEPENDS ${SRC_FILE}
- )
-
- target_sources(ic PRIVATE ${SSE_OUTPUT})
-
- if(USE_AVX2)
- add_custom_command(
- OUTPUT ${AVX2_OUTPUT}
- COMMAND ${CMAKE_C_COMPILER} ${AVX2} -DAVX2_ON ${avx2_c_flags_list} -c -o ${AVX2_OUTPUT} ${SRC_FILE}
- DEPENDS ${SRC_FILE}
- )
-
- target_sources(ic PRIVATE ${AVX2_OUTPUT})
- endif()
-endforeach()
-
-add_executable(test_bitd1unpack test_bitd1unpack.cpp)
-
-target_link_libraries(test_bitd1unpack ic)
-
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
- target_compile_options(test_bitd1unpack PRIVATE ${AVX2} -DAVX2_ON ${DEBUG})
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
- target_compile_options(test_bitd1unpack PRIVATE -march=armv8-a ${DEBUG})
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
- target_compile_options(test_bitd1unpack PRIVATE -mcpu=power9 -mtune=power9 -D__SSSE3__ ${DEBUG})
-endif()
-
-set(LIB_DESTINATION ../)
-
-install(TARGETS ic
- DESTINATION ${LIB_DESTINATION}
- COMPONENT ext)
-
-install(TARGETS test_bitd1unpack
- DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
- COMPONENT tests)
diff --git a/src/ext/for/README.md b/src/ext/for/README.md
deleted file mode 100644
index 31e8f25ebb4..00000000000
--- a/src/ext/for/README.md
+++ /dev/null
@@ -1,585 +0,0 @@
-TurboPFor: Fastest Integer Compression
-
-[//]: # ([![Build Status][travisBadge]][travisLink])
-[//]: # ([travisBadge]: https://api.travis-ci.com/powturbo/TurboPFor-Integer-Compression.svg?branch=master)
-[//]: # ([travisLink]: https://app.travis-ci.com/powturbo/TurboPFor-Integer-Compression)
-======================================
-* **TurboPFor: The synonym for "integer compression"**
- * **ALL** functions available for **AMD/Intel**, **64 bits ARMv8 NEON** Linux+MacOS/M1 & **Power9 Altivec**
- * 100% C (C++ headers), as simple as memcpy. OS:Linux amd64, arm64, Power9, MacOs (Amd/intel + Apple M1),
- * :+1: **Java** Critical Natives/JNI. Access TurboPFor **incl. SIMD/AVX2!** from Java as fast as calling from C
- * :sparkles: **FULL** range 8/16/32/64 bits scalar + 16/32/64 bits SIMD functions
- * No other "Integer Compression" compress/decompress faster
- * :sparkles: Direct Access, **integrated** (SIMD/AVX2) FOR/delta/Delta of Delta/Zigzag for sorted/unsorted arrays
- * **16 bits** + **64 bits** SIMD integrated functions
-* **For/PFor/PForDelta**
- * **Novel TurboPFor** (PFor/PForDelta) scheme w./ **direct access** + **SIMD/AVX2**. **+RLE**
- * Outstanding compression/speed. More efficient than **ANY** other fast "integer compression" scheme.
- * Compress 70 times faster and decompress up to 4 times faster than OptPFD
-* **Bit Packing**
- * Fastest and most efficient **"SIMD Bit Packing"** **15 Billions integers/sec (60Gb/s!)**
- * Scalar **"Bit Packing"** decoding nearly as fast as SIMD-Packing in realistic (No "pure cache") scenarios
- * **Direct/Random Access** : Access any single bit packed entry with **zero decompression**
-* **Variable byte**
- * Scalar **"Variable Byte"** faster and more efficient than **ANY** other implementation
- * SIMD **TurboByte** fastest group varint (16+32 bits) incl. integrated delta,zigzag,...
- * **TurboByte+TurboPackV** novel hybrid scheme combining the fastest SIMD codecs.
-* **Simple family**
- * **Novel** **"Variable Simple"** (incl. **RLE**) faster and more efficient than simple16, simple-8b
-* **Elias fano**
- * Fastest **"Elias Fano"** implementation w/ or w/o SIMD/AVX2
-+ **Transform**
- * Scalar & SIMD Transform: Delta, Zigzag, Zigzag of delta, XOR, Transpose/Shuffle,
- * **lossy** floating point compression with *TurboPFor* or [TurboTranspose](https://github.com/powturbo/TurboTranspose)+lz77
-* **Floating Point Compression**
- * Delta/Zigzag + improved gorilla style + (Differential) Finite Context Method FCM/DFCM floating point compression
- * Using **TurboPFor**, unsurpassed compression and more than 5 GB/s throughput
- * Point wise relative error bound **lossy** floating point compression
- * **TurboFloat** novel efficient floating point compression using TurboPFor
-* **Time Series Compression**
- * **Fastest Gorilla** 16/32/64 bits style compression (**zigzag of delta** + **RLE**).
- * can compress times series to only 0.01%. Speed > 10 GB/s compression and > 13 GB/s decompress.
-* **Inverted Index ...do less, go fast!**
- * Direct Access to compressed *frequency* and *position* data w/ zero decompression
- * **Novel** **"Intersection w/ skip intervals"**, decompress the minimum necessary blocks (**~10-15%)!**.
- * **Novel** Implicit skips with zero extra overhead
- * **Novel** Efficient **Bidirectional** Inverted Index Architecture (forward/backwards traversal) incl. "integer compression".
- * more than **2000! queries per second** on GOV2 dataset (25 millions documents) on a **SINGLE** core
- * :sparkles: Revolutionary Parallel Query Processing on Multicores **> 7000!!! queries/sec** on a simple quad core PC.<br>
- **...forget** ~~Map Reduce, Hadoop, multi-node clusters,~~ ...
-
-
-
-### Integer Compression Benchmark (single thread):
-- Download [IcApp](https://sites.google.com/site/powturbo/downloads) a new benchmark for TurboPFor<br>
- for testing allmost all integer and floating point file types.
-- Practical (No **PURE** cache) "integer compression" benchmark w/ **large** arrays.
-- [Benchmark Intel CPU: Skylake i7-6700 3.4GHz gcc 9.2](https://github.com/powturbo/TurboPFor/issues/47)
-- [Benchmark ARM: ARMv8 A73-ODROID-N2 1.8GHz](https://github.com/powturbo/TurboPFor/issues/49)
-
-##### - Synthetic data:
- - Generate and test (zipfian) skewed distribution (100.000.000 integers, Block size=128/256)<br>
- Note: Unlike general purpose compression, a small fixed size (ex. 128 integers) is in general used in "integer compression".
- Large blocks involved, while processing queries (inverted index, search engines, databases, graphs, in memory computing,...) need to be entirely decoded.
-
- ./icbench -a1.5 -m0 -M255 -n100M ZIPF
-
-|C Size|ratio%|Bits/Integer|C MB/s|D MB/s|Name 2019.11|
-|--------:|-----:|--------:|----------:|----------:|--------------|
-|62,939,886| 15.7| 5.04|**2369**|**10950**|**TurboPFor256**|
-|63,392,759| 15.8| 5.07|1359|7803|**TurboPFor128**|
-|63,392,801| 15.8| 5.07|1328|924|**TurboPForDA**|
-|65,060,504| 16.3| 5.20|60|2748|[FP_SIMDOptPFor](#FastPFor)|
-|65,359,916|16.3| 5.23| 32|2436|PC_OptPFD|
-|73,477,088|18.4| 5.88|408|2484|PC_Simple16|
-|73,481,096| 18.4| 5.88|624|8748|[FP_SimdFastPFor](#FastPFor) 64Ki *|
-|76,345,136| 19.1| 6.11|1072|2878|**VSimple**|
-|91,947,533| 23.0| 7.36|284|11737|[QMX](#QMX) 64k *|
-|93,285,864| 23.3| 7.46|1568|10232|[FP_GroupSimple](#FastPFor) 64Ki *|
-|95,915,096|24.0| 7.67| 848|3832|Simple-8b|
-|99,910,930| 25.0| 7.99|**17298**|**12408**|**TurboByte+TurboPack**|
-|99,910,930| 25.0| 7.99|**17357**|**12363**|**TurboPackV** sse|
-|99,910,930| 25.0| 7.99|11694|10138|**TurboPack** scalar|
-|99,910,930| 25.0| 7.99|8420|8876|**TurboFor**|
-|100,332,929| 25.1| 8.03|17077|11170|**TurboPack256V** avx2|
-|101,015,650| 25.3| 8.08|11191|10333|**TurboVByte**|
-|102,074,663| 25.5| 8.17|6689|9524|[MaskedVByte](#MaskedVByte)|
-|102,074,663| 25.5| 8.17|2260|4208|[PC_Vbyte](#PolyCom)|
-|102,083,036| 25.5| 8.17|5200|4268|[FP_VByte](#FastPFor)|
-|112,500,000| 28.1| 9.00|1528|12140|[VarintG8IU](#VarintG8IU)|
-|125,000,000| 31.2|10.00|13039|12366|**TurboByte**|
-|125,000,000| 31.2|10.00|11197|11984|[StreamVbyte 2019](#StreamVByte)|
-|400,000,000| 100.00| 32.00| 8960|8948|Copy|
-| | | | N/A | N/A |EliasFano|
-
-(*) codecs inefficient for small block sizes are tested with 64Ki integers/block.
-
-- MB/s: 1.000.000 bytes/second. **1000 MB/s = 1 GB/s**<br>
-- **#BOLD** = pareto frontier.<br>
-- FP=FastPFor SC:simdcomp PC:Polycom<br>
-- TurboPForDA,TurboForDA: Direct Access is normally used when accessing few individual values.<br>
-- Eliasfano can be directly used only for increasing sequences
-------------------------------------------------------------------------
-##### - Data files:
- - gov2.sorted from [DocId data set](#DocId) Block size=128/Delta coding
-
- ./icbench -fS -r gov2.sorted
-
-
-
-|Size |Ratio %|Bits/Integer|C Time MB/s|D Time MB/s|Function 2019.11|
-|-----------:|------:|-----:|-------:|-------:|---------------------|
-| 3,321,663,893| 13.9| 4.44|**1320**|**6088**|**TurboPFor**|
-| 3,339,730,557| 14.0| 4.47| 32| 2144|PC.OptPFD|
-| 3,350,717,959| 14.0| 4.48|**1536**|**7128**|**TurboPFor256**|
-| 3,501,671,314| 14.6| 4.68| 56| 2840|**VSimple**|
-| 3,768,146,467| 15.8| 5.04|**3228**| 3652|**EliasFanoV**|
-| 3,822,161,885| 16.0| 5.11| 572| 2444|PC_Simple16|
-| 4,411,714,936| 18.4| 5.90|**9304**|**10444**|**TurboByte+TurboPack**|
-| 4,521,326,518| 18.9| 6.05| 836| 3296|Simple-8b|
-| 4,649,671,427| 19.4| 6.22|3084| 3848|**TurboVbyte**|
-| 4,955,740,045| 20.7| 6.63|7064|10268|**TurboPackV**|
-| 4,955,740,045| 20.7| 6.63|5724| 8020|**TurboPack**|
-| 5,205,324,760| 21.8| 6.96|6952| 9488|SC_SIMDPack128|
-| 5,393,769,503| 22.5| 7.21|**14466**|**11902**|**TurboPackV256**|
-| 6,221,886,390| 26.0| 8.32|6668| 6952|**TurboFor**|
-| 6,221,886,390| 26.0| 8.32|6644| 2260|**TurboForDA**|
-| 6,699,519,000| 28.0| 8.96|1888| 1980|FP_Vbyte|
-| 6,700,989,563| 28.0| 8.96|2740| 3384|MaskedVByte|
-| 7,622,896,878| 31.9|10.20| 836| 4792|VarintG8IU|
-| 8,060,125,035| 33.7|11.50|8456| 9476|Streamvbyte 2019|
-| 8,594,342,216| 35.9|11.50|5228| 6376|libfor|
-|23,918,861,764|100.0|32.00|5824| 5924|Copy|
-
-Block size: 64Ki = 256k bytes. Ki=1024 Integers
-
-|Size |Ratio %|Bits/Integer|C Time MB/s|D Time MB/s|Function |
-|----------:|-----:|----:|------:|------:|---------------------|
-| 3,164,940,562| 13.2|**4.23**|**1344**|**6004**|**TurboPFor 64Ki**|
-| 3,273,213,464| 13.7| 4.38|**1496**|**7008**|**TurboPFor256 64Ki**|
-| 3,965,982,954| 16.6| 5.30|**1520**| 2452|[lz4](#lz4)+DT 64Ki|
-| 4,234,154,427| 17.7| 5.66| 436| 5672|qmx 64Ki|
-| 6,074,995,117| 25.4| 8.13| 1976| 2916|[blosc_lz4](#blosc) 64Ki|
-| 8,773,150,644| 36.7|11.74| 2548|5204|blosc_lz 64Ki|
-
-"lz4+DT 64Ki" = Delta+Transpose from TurboPFor + lz4<br>
-"blosc_lz4" internal lz4 compressor+vectorized shuffle
-
-##### - Time Series:
-- Test file [Timestamps: ts.txt(sorted)](https://github.com/zhenjl/encoding/tree/master/benchmark/data)
-
- ./icapp -Ft ts.txt -I15 -J15
-
-|Function |C MB/s| size |ratio%| D MB/s|Text
-|----------------|-----:|--------:|------:|------:|--------------------|
-|bvzenc32 |**10632**|45,909|0.008|**12823**|ZigZag|
-|bvzzenc32 |**8914**|56,713|0.010|**13499**|ZigZag Delta of delta|
-|vsenc32 |**12294**|140,400| 0.024 |12877 |Variable Simple|
-|p4nzenc256v32 | 1932| 596,018| 0.10 |13326 |TurboPFor256 ZigZag|
-|p4ndenc256v32 | 1961| 596,018| 0.10 |13339 |TurboPFor256 Delta|
-|bitndpack256v32 |**12564**|909,189| 0.16 |13505 |TurboPackV256 Delta|
-|p4nzenc32 | 1810| 1,159,633| 0.20 | 8502 |TurboPFor ZigZag|
-|p4nzenc128v32 | 1795| 1,159,633| 0.20 |13338 |TurboPFor ZigZag|
-|bitnzpack256v32 | 9651| 1,254,757| 0.22 |**13503**|TurboPackV256 ZigZag|
-|bitnzpack128v32 |10155| 1,472,804| 0.26 |13380 |TurboPackV ZigZag|
-|vbddenc32 | 6198| 18,057,296| 3.13 |10982 |TurboVByte Delta of delta|
-|memcpy |13397|577,141,992|100.00||
-
-##### - Transpose/Shuffle (no compression)
- ./icbench -eTRANSFORM ZIPF
-
-|Size |C Time MB/s|D Time MB/s|Function|
-|----------:|------:|------:|-----------------------------------|
-|100,000,000|**9400**|**9132**|**TPbyte 4** TurboPFor Byte Transpose/shuffle AVX2|
-|100,000,000|8784|8860|**TPbyte 4** TurboPFor Byte Transpose/shuffle SSE|
-|100,000,000|7688|7656|Blosc_Shuffle AVX2|
-|100,000,000|**5204**|**7460**|**TPnibble 4** TurboPFor Nibble Transpose/shuffle SSE|
-|100,000,000|6620|6284|Blosc shuffle SSE|
-|100,000,000|3156|3372|Bitshuffle AVX2|
-|100,000,000|2100|2176|Bitshuffle SSE|
-
-##### - (Lossy) Floating point compression:
- ./icapp -Fd file " 64 bits floating point raw file
- ./icapp -Ff file " 32 bits floating point raw file
- ./icapp -Fcf file " text file with miltiple entries (ex. 8.657,56.8,4.5 ...)
- ./icapp -Ftf file " text file (1 entry per line)
- ./icapp -Ftf file -v5 " + display the first entries read
- ./icapp -Ftf file.csv -K3 " but 3th column in a csv file (ex. number,Text,456.5 -> 456.5
- ./icapp -Ftf file -g.001 " lossy compression with allowed pointwise relative error 0.001
-
-- see also [TurboTranspose](https://github.com/powturbo/TurboTranspose)
-
-##### - Compressed Inverted Index Intersections with GOV2<br />
- GOV2: 426GB, 25 Millions documents, average doc. size=18k.
-
- + Aol query log: 18.000 queries<br />
- **~1300** queries per second (single core)<br />
- **~5000** queries per second (quad core)<br />
- Ratio = 14.37% Decoded/Total Integers.
-
- + TREC Million Query Track (1MQT):<br />
- **~1100** queries per second (Single core)<br />
- **~4500** queries per second (Quad core CPU)<br />
- Ratio = 11.59% Decoded/Total Integers.
-
-- Benchmarking intersections (Single core, AOL query log)
-
-| max.docid/q|Time s| q/s | ms/q | % docid found|
-|-----------------:|---:|----:|-----:|-------:|
-|1.000|7.88|2283.1|0.438|81|
-|10.000|10.54|1708.5|0.585|84|
-| ALL |13.96|1289.0|0.776|100|
-q/s: queries/second, ms/q:milliseconds/query
-
-- Benchmarking Parallel Query Processing (Quad core, AOL query log)
-
-| max.docid/q|Time s| q/s | ms/q | % docids found|
-|-----------------:|----:|----:|-----:|-------:|
-|1.000|2.66|6772.6|0.148|81|
-|10.000|3.39|5307.5|0.188|84|
-|ALL|3.57|5036.5|0.199|100|
-
-###### Notes:
-- Search engines are spending 90% of the time in intersections when processing queries.
-- Most search engines are using pruning strategies, caching popular queries,... to reduce the time for intersections and query processing.
-- As indication, google is processing [40.000 Queries per seconds](http://www.internetlivestats.com/google-search-statistics/),
-using [900.000 multicore servers](https://www.cloudyn.com/blog/10-facts-didnt-know-server-farms/) for searching [8 billions web pages](http://searchenginewatch.com/sew/study/2063479/coincidentally-googles-index-size-jumps) (320 X size of GOV2).
-- Recent "integer compression" GOV2 experiments (best paper at ECIR 2014) [On Inverted Index Compression for Search Engine Efficiency](http://www.dcs.gla.ac.uk/~craigm/publications/catena14compression.pdf) using 8-core Xeon PC are reporting 1.2 seconds per query (for 1.000 Top-k docids).
-
-### Compile:
- Download or clone TurboPFor
- git clone git://github.com/powturbo/TurboPFor.git
- cd TurboPFor
- make
-
-
- To benchmark external libraries + lz77 compression:
- git clone --recursive git://github.com/powturbo/TurboPFor.git
- cd TurboPFor
- make CODEC1=1 CODEC2=1 LZ=1
-
-###### Windows visual c++
- nmake /f makefile.vs
-
-###### Windows visual studio c++
- project files under vs/vs2017
-
-### Testing:
-##### - Synthetic data (use ZIPF parameter):
- + benchmark groups of "integer compression" functions <br />
-
- ./icbench -eBENCH -a1.2 -m0 -M255 -n100M ZIPF
- ./icbench -eBITPACK/VBYTE -a1.2 -m0 -M255 -n100M ZIPF
-
- >*Type "icbench -l1" for a list*
-
- >*-zipfian distribution alpha = 1.2 (Ex. -a1.0=uniform -a1.5=skewed distribution)<br />
- -number of integers = 100.000.000<br />
- -integer range from 0 to 255<br />*
-
- + Unsorted lists: individual function test (ex. Copy TurboPack TurboPFor)<br />
-
- ./icbench -a1.5 -m0 -M255 -ecopy/turbopack/turbopfor/turbopack256v ZIPF
-
- + Unsorted lists: Zigzag encoding w/ option **-fz** or FOR encoding<br />
-
- ./icbench -fz -eturbovbyte/turbopfor/turbopackv ZIPF
- ./icbench -eturboforv ZIPF
-
- + Sorted lists: differential coding w/ option **-fs** (increasing) or **-fS** (strictly increasing)<br />
-
- ./icbench -fs -eturbopack/turbopfor/turbopfor256v ZIPF
-
- + Generate interactive "file.html" plot for browsing
-
- ./icbench -p2 -S2 -Q3 file.tbb
-
- + Unit test: test function from bit size 0 to 32
-
- ./icbench -m0 -M32 -eturbpfor -fu
- ./icbench -m0 -M8 -eturbopack -fs -n1M
-
-##### - Data files:
- - Raw 32 bits binary data file [Test data](https://github.com/ot/partitioned_elias_fano/tree/master/test/test_data)
-
- ./icbench file
- ./icapp file
- ./icapp -Fs file "16 bits raw binary file
- ./icapp -Fu file "32 bits raw binary file
- ./icapp -Fl file "64 bits raw binary file
- ./icapp -Ff file "32 bits raw floating point binary file
- ./icapp -Fd file "64 bits raw floating point binary file
-
- - Text file: 1 entry per line. [Test data: ts.txt(sorted) and lat.txt(unsorted)](https://github.com/zhenjl/encoding/tree/master/benchmark/data))
-
- ./icbench -eBENCH -fts ts.txt
- ./icbench -eBENCH -ft lat.txt
-
- ./icapp -Fts data.txt "text file, one 16 bits integer per line
- ./icapp -Ftu ts.txt "text file, one 32 bits integer per line
- ./icapp -Ftl ts.txt "text file, one 64 bits integer per line
- ./icapp -Ftf file "text file, one 32 bits floating point (ex. 8.32456) per line
- ./icapp -Ftd file "text file, one 64 bits floating point (ex. 8.324567789) per line
- ./icapp -Ftd file -v5 "like prev., display the first 100 values read
- ./icapp -Ftd file -v5 -g.00001 "like prev., error bound lossy floating point compression
- ./icapp -Ftt file "text file, timestamp in seconds iso-8601 -> 32 bits integer (ex. 2018-03-12T04:31:06)
- ./icapp -FtT file "text file, timestamp in milliseconds iso-8601 -> 64 bits integer (ex. 2018-03-12T04:31:06.345)
- ./icapp -Ftl -D2 -H file "skip 1th line, convert numbers with 2 decimal digits to 64 bits integers (ex. 456.23 -> 45623)
- ./icapp -Ftl -D2 -H -K3 file.csv "like prev., use the 3th number in the line (ex. label=3245, text=99 usage=456.23 -> 456.23 )
- ./icapp -Ftl -D2 -H -K3 -k| file.csv "like prev., use '|' as separator
-
- - Text file: multiple numbers separated by non-digits (0..9,-,.) characters (ex. 134534,-45678,98788,4345, )
-
- ./icapp -Fc data.txt "text file, 32 bits integers (ex. 56789,3245,23,678 )
- ./icapp -Fcd data.txt "text file, 64 bits floting-point numbers (ex. 34.7689,5.20,45.789 )
-
- - Multiblocks of 32 bits binary file. (Example gov2 from [DocId data set](#DocId))<br />
- Block format: [n1: #of Ids][Id1] [Id2]...[IdN] [n2: #of Ids][Id1][Id2]...[IdN]...
-
- ./icbench -fS -r gov2.sorted
-
-
-##### - Intersections:
- 1 - Download Gov2 (or ClueWeb09) + query files (Ex. "1mq.txt") from [DocId data set](#DocId)<br />
- 8GB RAM required (16GB recommended for benchmarking "clueweb09" files).
-
- 2 - Create index file
-
-
- ./idxcr gov2.sorted .
-
-
- >*create inverted index file "gov2.sorted.i" in the current directory*
-
- 3 - Test intersections
-
-
- ./idxqry gov2.sorted.i 1mq.txt
-
-
- >*run queries in file "1mq.txt" over the index of gov2 file*
-
-##### - Parallel Query Processing:
- 1 - Create partitions
-
-
- ./idxseg gov2.sorted . -26m -s8
-
-
- >*create 8 (CPU hardware threads) partitions for a total of ~26 millions document ids*
-
- 2 - Create index file for each partition
-
-
- ./idxcr gov2.sorted.s*
-
-
- >*create inverted index file for all partitions "gov2.sorted.s00 - gov2.sorted.s07" in the current directory*
-
- 3 - Intersections:
-
- delete "idxqry.o" file and then type "make para" to compile "idxqry" w. multithreading
-
-
- ./idxqry gov2.sorted.s*.i 1mq.txt
-
- >*run queries in file "1mq.txt" over the index of all gov2 partitions "gov2.sorted.s00.i - gov2.sorted.s07.i".*
-
-### Function usage:
-See benchmark "icbench" program for "integer compression" usage examples.
-In general encoding/decoding functions are of the form:
-
- >**char *endptr = encode( unsigned *in, unsigned n, char *out, [unsigned start], [int b])**<br />
- endptr : set by encode to the next character in "out" after the encoded buffer<br />
- in : input integer array<br />
- n : number of elements<br />
- out : pointer to output buffer<br />
- b : number of bits. Only for bit packing functions<br />
- start : previous value. Only for integrated delta encoding functions
-
-
- >**char *endptr = decode( char *in, unsigned n, unsigned *out, [unsigned start], [int b])**<br />
- endptr : set by decode to the next character in "in" after the decoded buffer<br />
- in : pointer to input buffer<br />
- n : number of elements<br />
- out : output integer array<br />
- b : number of bits. Only for bit unpacking functions<br />
- start : previous value. Only for integrated delta decoding functions
-
- **Simple high level functions:**
- >**size_t compressed_size = encode( unsigned *in, size_t n, char *out)**<br />
- compressed_size : number of bytes written into compressed output buffer out<br />
-
- >**size_t compressed_size = decode( char *in, size_t n, unsigned *out)**<br />
- compressed_size : number of bytes read from compressed input buffer in<br />
-
-### Function syntax:
- - {vb | p4 | bit | vs}[n][d | d1 | f | fm | z ]{enc/dec | pack/unpack}[| 128V | 256V][8 | 16 | 32 | 64]:<br />
- vb: variable byte<br />
- p4: turbopfor<br />
- vs: variable simple<br />
- bit: bit packing<br />
- n : high level array functions for large arrays.
-
- '' : encoding for unsorted integer lists<br />
- 'd' : delta encoding for increasing integer lists (sorted w/ duplicate)<br />
- 'd1': delta encoding for strictly increasing integer lists (sorted unique)<br />
- 'f' : FOR encoding for sorted integer lists<br />
- 'z' : ZigZag encoding for unsorted integer lists<br />
-
- 'enc' or 'pack' : encode or bitpack<br />
- 'dec' or 'unpack': decode or bitunpack<br />
- 'NN' : integer size (8/16/32/64)<br />
-
-header files to use with documentation:<br />
-
-| c/c++ header file|Integer Compression functions| examples |
-|------------|-----------------------------|-----------------|
-|vint.h|variable byte| vbenc32/vbdec32 vbdenc32/vbddec32 vbzenc32/vbzdec32 |
-|vsimple.h|variable simple| vsenc64/vsdec64 |
-|vp4.h|TurboPFor| p4enc32/p4dec32 p4denc32/p4ddec32 p4zenc32/p4zdec32 |
-|bitpack.h|Bit Packing, For, +Direct Access| bitpack256v32/bitunpack256v32 bitforenc64/bitfordec64|
-|eliasfano.h|Elias Fano| efanoenc256v32/efanoc256v32 |
-
-Note: Some low level functions (like p4enc32) are limited to 128/256 (SSE/AVX2) integers per call.
-
-### Environment:
-###### OS/Compiler (64 bits):
-- Windows: MinGW-w64 makefile
-- Windows: Visual c++ (>=VS2008) - makefile.vs (for nmake)
-- Windows: Visual Studio project file - vs/vs2017 - Thanks to [PavelP](https://github.com/pps83)
-- Linux amd64: GNU GCC (>=4.6)
-- Linux amd64: Clang (>=3.2)
-- Linux arm64: 64 bits aarch64 ARMv8: gcc (>=6.3)
-- Linux arm64: 64 bits aarch64 ARMv8: clang
-- MaxOS: XCode (>=9)
-- MaxOS: Apple M1 (Clang)
-- PowerPC ppc64le (incl. SIMD): gcc (>=8.0)
-
-###### Multithreading:
-- All TurboPFor integer compression functions are thread safe
-
-### References:
-
-* [TurboPFor: an analysis](https://michael.stapelberg.ch/posts/2019-02-05-turbopfor-analysis/)
-
-* **Applications:**
- * [Debian Code Search](https://github.com/Debian/dcs/)</br>
- [Debian Code Search: positional index, TurboPFor-compressed](https://michael.stapelberg.ch/posts/2019-09-29-dcs-positional-turbopfor-index/)
- * [Graph500](https://github.com/julianromera/graph500)
- * [Small Polygon Compression](https://arxiv.org/abs/1509.05505) + [Poster](http://abhinavjauhri.me/publications/dcc_poster_2016.pdf) + [code](https://github.com/ajauhri/bignum_compression)
- * [Parallel Graph Analysis (Lecture 18)](http://www.cs.rpi.edu/~slotag/classes/FA16/) + [code](http://www.cs.rpi.edu/~slotag/classes/FA16/handson/lec18-comp2.cpp)
-
-* **Benchmark references:**
- * <a name="FastPFor"></a>[FastPFor](https://github.com/lemire/FastPFor) + [Simdcomp](https://github.com/lemire/simdcomp): SIMDPack FPF, Vbyte FPF, VarintG8IU, StreamVbyte, GroupSimple
- * <a name="OptPFD"></a><a name="Simple16"></a>[Optimized Pfor-delta compression code](http://jinruhe.com): OptPFD/OptP4, Simple16 (limited to 28 bits integers)
- * <a name="MaskedVByte"></a>[MaskedVByte](http://maskedvbyte.org/). See also: [Vectorized VByte Decoding](http://engineering.indeed.com/blog/2015/03/vectorized-vbyte-decoding-high-performance-vector-instructions/)
- * <a name="Streamvbyte"></a>[Streamvbyte](https://github.com/lemire/streamvbyte).
- * <a name="Simple-8b"></a>[Index Compression Using 64-Bit Words](http://people.eng.unimelb.edu.au/ammoffat/abstracts/am10spe.html): Simple-8b (speed optimized version tested)
- * <a name="libfor"></a>[libfor](https://github.com/cruppstahl/for)
- * <a name="QMX"></a>[Compression, SIMD, and Postings Lists](http://www.cs.otago.ac.nz/homepages/andrew/papers/) QMX integer compression from the "simple family"
- * <a name="lz4"></a>[lz4](https://github.com/Cyan4973/lz4). included w. block size 64K as indication. Tested after preprocessing w. delta+transpose
- * <a name="blosc"></a>[blosc](https://github.com/Blosc/c-blosc). blosc is like transpose/shuffle+lz77. Tested blosc+lz4 and blosclz incl. vectorizeed shuffle.<br>
- * <a name="DocId"></a>[Document identifier data set](http://lemire.me/data/integercompression2014.html)
-
-* **Integer compression publications:**
- * :green_book:[Evaluating Lightweight Integer Compression Algorithms in Column-Oriented In-Memory DBMS](http://www.adms-conf.org/2021-camera-ready/heinzl_adms21.pdf)
- * :green_book:[In Vacuo and In Situ Evaluation of SIMD Codecs (TurboPackV,TurboPFor/QMX)](http://dl.acm.org/citation.cfm?id=3015023) + [paper](http://www.cs.otago.ac.nz/homepages/andrew/papers/)
- * :green_book:[SIMD Compression and the Intersection of Sorted Integers](http://arxiv.org/abs/1401.6399)
- * :green_book:[Partitioned Elias-Fano Indexes](http://www.di.unipi.it/~ottavian/files/elias_fano_sigir14.pdf)
- * :green_book:[On Inverted Index Compression for Search Engine Efficiency](http://www.dcs.gla.ac.uk/~craigm/publications/catena14compression.pdf)
- * :green_book:[Google's Group Varint Encoding](http://static.googleusercontent.com/media/research.google.com/de//people/jeff/WSDM09-keynote.pdf)
- * :green_book:[Integer Compression tweets](https://twitter.com/search?q=%23integercompression&src=typd)
- * :green_book:[Efficient Compression of Scientific Floating-Point Data and An Application in Structural Analysis](https://www.jstage.jst.go.jp/article/jsces/2017/0/2017_20170002/_article)
- * :green_book:[SPDP is a compression/decompression algorithm for binary IEEE 754 32/64 bits floating-point data](http://cs.txstate.edu/~burtscher/research/SPDPcompressor/)<br />
- :green_book:[ SPDP - An Automatically Synthesized Lossless Compression Algorithm for Floating-Point Data](http://cs.txstate.edu/~mb92/papers/dcc18.pdf) + [DCC 2018](http://www.cs.brandeis.edu//~dcc/Programs/Program2018.pdf)
-
-Last update: 13 Nov 2021
-
-## APPENDIX: icbench Integer Compression Benchmark
-
-##### TurboPFor + external libraries
-<pre>
-TurboPFor https://github.com/powturbo/TurboPFor
-FastPFor (FP) https://github.com/lemire/FastPFor
-lz4 https://github.com/Cyan4973/lz4
-LittleIntPacker (LI) https://github.com/lemire/LittleIntPacker
-MaskedVbyte http://maskedvbyte.org
-Polycom (PC) https://github.com/encode84/bcm
-simdcomp (SC) https://github.com/lemire/simdcomp
-Simple-8b optimized https://github.com/powturbo/TurboPFor
-Streamvbyte https://github.com/lemire/streamvbyte
-VarintG8IU https://github.com/lemire/FastPFor
-</pre>
-
-##### Functions integrated into 'icbench' for benchmarking
-<pre>
-Codec group:
-TURBOPFOR TurboPFor library TurboPFor256V/TurboPack256V/TurboPFor256N/TurboPFor/TurboPackV/TurboVByte/TurboPack/TurboForDA/EliasFano/VSimple/TurboPForN/TurboPackN/TurboPForDI
-DEFAULT Default TurboPFor/TurboPackV/TurboVByte/TurboPack/TurboFor/TurboPForN/TurboPackN/TurboPForDI/TurboPFor256V/TurboPack256V/TurboPFor256N
-BENCH Benchmark TurboPFor/TurboPackV/TurboVByte/TurboPack/QMX/FP.SimdFastPfor/FP.SimdOptPFor/MaskedVbyte/StreamVbyte
-EFFICIENT Efficient TurboPFor/vsimple/turbovbyte
-TRANSFORM transpose/shufle,delta,zigzag tpbyte4s/tpbyte,4/tpnibble,4/ZigZag_32/Delta_32/BitShuffle,4
-BITPACK Bit Packing TurboPack256V/TurboPackV/TurboPackH/TurboPack/SC.SimdPack128/SC.SimdPack256
-VBYTE Variable byte TurboVByte/FP.VByte/PC.Vbyte/VarintG8IU/MaskedVbyte/StreamVbyte
-SIMPLE Simple Family simple8b/simple16/vsimple/qmx
-LZ4 lz4+bitshufle/transpose 4,8 lz4_bitshufle/lz4_tp4/lz4_tp8
-LI Little Integer LI_Pack/LI_TurboPack/LI_SuperPack/LI_HorPack
-
-
-Function Description level
-
--------- ----------- -----
-TurboPFor PFor (SSE2)
-TurboPForN PFor (SSE2) large blocks
-TurboPFor256 PFor (AVX2)
-TurboPFor256N PFor (AVX2) large blocks
-TurboPForDA PFor direct access
-TurboPForDI PFord min
-TurboPForZZ PFor zigzag of delta
-TurboFor FOR
-TurboForV FOR (SIMD)
-TurboFor256V FOR (AVX2)
-TurboForDA FOR direct access
-TurboPackDA Bit packing direct access
-TurboPack Bit packing (scalar)
-TurboPackN Bit packing (scalar) large blocks
-TurboPackV Bit packing (SSE2 Vertical)
-TurboPackH Bit packing (SSE2 Horizontal)
-TurboPackVN Bit packing (SSE2 large block)
-TurboPack256V Bit packing (AVX2 Vertical)
-TurboPack256N Bit packing (AVX2 large block)
-TurboVByte Variable byte (scalar)
-VSimple Variable simple (scalar)
-EliasFano Elias fano (scalar)
-EliasFanoV Eliasfano (SSE2)
-EliasFano256V Elias fano (AVX2)
-memcpy memcpy
-copy Integer copy
-tpbyte4s Byte Transpose (scalar)
-tpbyte Byte transpose (simd) 2,4,8
-tpnibble Nibble transpose (simd) 2,4,8
-ZigZag32 ZigZag encoding (sse2)
-Delta32 Delta encoding (sse2)
-DDelta32 Delta of delta encoding (sse2)
-Xor32 Xor encoding (sse2)
-FP_PREV64 Floating point PFOR
-FP_FCM64 Floating point PFOR (FCM)
-FP_DFCM64 Floating point PFOR (DFCM)
-TurboPFor64 PFOR 64
-TurboPFor64V PFOR 64
-Simple8b 64 bits Simple family (instable)
-PC_Simple16 Simple 16. limited to 28 bits
-PC_OptPFD OptPFD. limited to 28 bits
-PC_Vbyte Variable byte
-PC_Rice Rice coding (instable)
-VarintG8IU Variable byte SIMD
-MaskedVbyte Variable byte SIMD
-StreamVbyte Variable byte SIMD
-FP_FastPFor PFor scalar (inefficient for small blocks)
-FP_SimdFastPFor PFor SIMD (inefficient for small blocks)
-FP_OptPFor OptPFor scalar
-FP_SIMDOptPFor OptPFor SIMD
-FP_VByte Variable byte
-FP_Simple8bRLE Simple-8b + rle
-FP_GROUPSIMPLE Group Simple
-SC_SIMDPack128 Bit packing (SSE4.1)
-SC_SIMDPack256 Bit packing (SSE4.1)
-SC_For For (SSE4.1)
-SC_ForDA For direct access (SSE4.1)
-LibFor_For For
-LibFor_ForDA For direct access
-LI_Pack Bit packing (scalar)
-LI_TurboPack Bit packing (scalar)
-LI_SuperPack Bit packing (scalar)
-LI_HorPack Bit packing (sse4.1 horizontal)
-LI_BMIPack256 Bit packing (avx2)
-lz4 lz4
-lz4_bit Bitshuffle + [delta]+lz4 2,4,8
-lz4_nibble TurboPFor's [delta]+nibble transpose + lz4 2,4,8
-lz4_bitxor Bitshuffle + [xor]+lz4 2,4,8
-lz4_nibblexor TurboPFor's [xor]+nibble transpose + lz4 2,4,8
-lz4_byte TurboPFor's [delta]+byte transpose + lz4 2,4,8
-BitShuffle Bit shuffle (simd) 2,4,8
-</pre>
-
diff --git a/src/ext/for/bitpack.c b/src/ext/for/bitpack.c
deleted file mode 100644
index f7115dbc340..00000000000
--- a/src/ext/for/bitpack.c
+++ /dev/null
@@ -1,428 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// "Integer Compression" bit packing
-
-#include <stdio.h>
-#define BITUTIL_IN
-#define VINT_IN
-#include "conf.h"
-#include "bitutil.h"
-#include "vint.h"
-#include "bitpack.h"
-#define PAD8(_x_) ( (((_x_)+8-1)/8) )
-
- #ifdef __ARM_NEON
-#define PREFETCH(_ip_,_rw_)
- #else
-#define PREFETCH(_ip_,_rw_) __builtin_prefetch(_ip_,_rw_)
- #endif
-
-#pragma warning( disable : 4005)
-#pragma warning( disable : 4090)
-#pragma warning( disable : 4068)
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunsequenced"
-
-#ifndef __AVX2__ //----------------------------------- Plain -----------------------------------------------------------------------
-typedef unsigned char *(*BITPACK_F8)( uint8_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D8)( uint8_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint8_t start);
-typedef unsigned char *(*BITPACK_F16)(uint16_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D16)(uint16_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint16_t start);
-typedef unsigned char *(*BITPACK_F32)(uint32_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D32)(uint32_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint32_t start);
-typedef unsigned char *(*BITPACK_F64)(uint64_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D64)(uint64_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint64_t start);
-
-#if 1 //def _MSC_VER
-#define VX (v=x)
-#define V x
-#else
-#define VX v
-#define V v
-#endif
-
-#if 0
-#define IP0(_ip_,_x_) *_ip_
-#define IP( _ip_,_x_) *_ip_++
-#define IPI(_ip_)
-#else
-#define IP0(_ip_,_x_) _ip_[_x_]
-#define IP( _ip_,_x_) _ip_[_x_]
-#define IPI(_ip_) _ip_ += 32
-#endif
-
-#define IP9(_ip_,_x_, _parm_)
-#define IPW(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_))
-
-#define IPV(_ip_,_x_) IP(_ip_,_x_)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define DELTA
-
-#define IP9(_ip_,_x_, _parm_) V = IP0(_ip_,_x_) - start; start = IP(_ip_,_x_)
-#define IPV(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start)
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitdpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IP9(_ip_,_x_, _parm_)
-#define IPV(_ip_,_x_) IP(_ip_,_x_) - start
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitfpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IP9( _ip_,_x_, _parm_) V = IP0(_ip_,_x_) - start - 1; start = IP(_ip_,_x_)
-#define IPV( _ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start - 1)
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitd1pack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-/*#define IP9( _ip_,_x_, _parm_) v = IP(_ip_,_x_) - start - mdelta; start = IP(_ip_,_x_)
-#define IPV( _ip_,_x_) v
-#define IPX(_ip_,_x_) (v = IP(_ip_,_x_) - start - mdelta)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitepack
-#include "bitpack_.h"*/
-
-#define IP9(_ip_,_x_, _parm_) V = TEMPLATE2(zigzagenc, USIZE)(IP(_ip_,_x_) - start); start = IP(_ip_,_x_)
-#define IPV(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = TEMPLATE2(zigzagenc, USIZE)(IP(_ip_,_x_) - start))
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitzpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IPI(_ip_) _ip_ += 32; start += 32
-#define IP9(_ip_,_x_, _parm_)
-#define IPV(_ip_,_x_) (IP(_ip_,_x_) - start - (_x_) - 1)
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start - (_x_) - 1)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitf1pack
-#include "bitpack_.h"
-#undef IPI
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define BITNPACK(in, n, out, _csize_, _usize_) { unsigned char *op = out;\
- for(ip = in, in += n; ip < in;) { \
- TEMPLATE3(uint, _usize_, _t) o,x;\
- unsigned iplen = in - ip,b; \
- if(iplen > _csize_) iplen = _csize_; \
- PREFETCH(ip+512,0);\
- o = TEMPLATE2(bit,_usize_)(ip, iplen, &x); b = TEMPLATE2(bsr,_usize_)(o);\
- *op++ = b; op = TEMPLATE2(bitpacka, _usize_)[b](ip, iplen, op);\
- ip += iplen;\
- }\
- return op - out;\
-}
-
-#define BITNDPACK(in, n, out, _csize_, _usize_, _bitd_, _bitpacka_) { if(!n) return 0;\
- unsigned char *op = out; \
- TEMPLATE3(uint, _usize_, _t) o,x;\
- start = *in++; \
- TEMPLATE2(vbxput, _usize_)(op, start);\
- for(n--,ip = in; ip != in + (n&~(_csize_-1)); ) { unsigned b; PREFETCH(ip+512,0);\
- o = TEMPLATE2(_bitd_, _usize_)(ip, _csize_, &x, start); b = TEMPLATE2(bsr,_usize_)(o); *op++ = b; op = TEMPLATE2(_bitpacka_,_usize_)[b](ip, _csize_, op, start); ip += _csize_; start = ip[-1];\
- }\
- if(n&=(_csize_-1)) { unsigned b;\
- o = TEMPLATE2(_bitd_, _usize_)(ip, n, &x, start); b = TEMPLATE2(bsr,_usize_)(o); *op++ = b; op = TEMPLATE2(_bitpacka_,_usize_)[b](ip, n, op, start);\
- }\
- return op - out;\
-}
-
-size_t bitnpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNPACK(in, n, out, 128, 8); }
-size_t bitnpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNPACK(in, n, out, 128, 16); }
-size_t bitnpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNPACK(in, n, out, 128, 32); }
-size_t bitnpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNPACK(in, n, out, 128, 64); }
-
-size_t bitndpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitd, bitdpacka); }
-size_t bitndpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitd, bitdpacka); }
-size_t bitndpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitd, bitdpacka); }
-size_t bitndpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitd, bitdpacka); }
-
-size_t bitnd1pack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitd1, bitd1packa); }
-size_t bitnd1pack16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitd1, bitd1packa); }
-size_t bitnd1pack32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitd1, bitd1packa); }
-size_t bitnd1pack64(uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitd1, bitd1packa); }
-
-size_t bitnzpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitz, bitzpacka); }
-size_t bitnzpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitz, bitzpacka); }
-size_t bitnzpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitz, bitzpacka); }
-size_t bitnzpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitz, bitzpacka); }
-
-size_t bitnfpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitf, bitfpacka); }
-size_t bitnfpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitf, bitfpacka); }
-size_t bitnfpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitf, bitfpacka); }
-size_t bitnfpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitf, bitfpacka); }
-
-#endif // ifndef AVX2
-
-//--------------------------------------- SIMD ----------------------------------------------------------------------------------------------
-
-#define _BITNPACKV(in, n, out, _csize_, _usize_, _bitpackv_) {\
- unsigned char *op = out; TEMPLATE3(uint, _usize_, _t) _o,_x;\
- for(ip = in; ip != in + (n&~(_csize_-1)); ip += _csize_) { PREFETCH(ip+512,0);\
- unsigned _b; _o = TEMPLATE2(bit,_usize_)(ip, _csize_, &_x); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpackv_, _usize_)(ip, _csize_, op, _b);\
- } if(n&=(_csize_-1)) { unsigned _b; _o = TEMPLATE2(bit,_usize_)(ip, n, &_x); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(bitpack, _usize_)(ip, n, op, _b); }\
- return op - out;\
-}
-
-#define _BITNDPACKV(in, n, out, _csize_, _usize_, _bitdv_, _bitpackv_, _bitd_, _bitpack_) { if(!n) return 0;\
- unsigned char *op = out; TEMPLATE3(uint, _usize_, _t) _o,_x;\
- start = *in++; \
- TEMPLATE2(vbxput, _usize_)(op, start);\
- for(n--,ip = in; ip != in + (n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned _b; _o = TEMPLATE2(_bitdv_, _usize_)(ip, _csize_, &_x, start); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpackv_, _usize_)(ip, _csize_, op, start, _b); ip += _csize_; start = ip[-1];\
- } if(n&=(_csize_-1)) { unsigned _b; _o = TEMPLATE2(_bitd_, _usize_)(ip, n, &_x, start); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpack_, _usize_)(ip, n, op, start, _b); }\
- return op - out;\
-}
-
- #ifdef __AVX2__
-#include <immintrin.h>
-#include "bitpack_.h"
-
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define PAD8(__x) (((__x)+8-1)/8)
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define VI32(ip, i, iv, parm)
-#define IP32(ip, i, iv) _mm256_loadu_si256(ip++)
-
-unsigned char *bitpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(256*b); BITPACK256V32(in, b, out, 0); return pout; }
-#undef VI32
-#undef IP32
-
-
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm256_sub_epi32(_mm256_loadu_si256(_ip_++),sv)
-#define IP32(_ip_, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitfpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm256_sub_epi32(_mm256_loadu_si256(_ip_++),_sv_); _sv_ = _mm256_add_epi32(_sv_,cv);
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitf1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set_epi32(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1), cv = _mm256_set1_epi32(8);
- BITPACK256V32(in, b, out, sv); return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = mm256_delta_epi32(v,_sv_); _sv_ = v
-#define IP32(ip, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitdpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v,sv = _mm256_set1_epi32(start);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = _mm256_sub_epi32(mm256_delta_epi32(v,_sv_),cv); _sv_ = v
-unsigned char *bitd1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start), cv = _mm256_set1_epi32(1);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = mm256_delta_epi32(v,_sv_); _sv_ = v; _iv_ = mm256_zzage_epi32(_iv_)
-unsigned char *bitzpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start), cv = _mm256_set1_epi32(1);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-size_t bitnpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 256, 32, bitpack256v); }
-size_t bitndpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitd, bitdpack256v, bitd, bitdpack); }
-size_t bitnd1pack256v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitd1, bitd1pack256v,bitd1, bitd1pack); }
-size_t bitnzpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitz, bitzpack256v, bitz, bitzpack); }
-size_t bitnfpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitf, bitfpack256v, bitf, bitfpack); }
-
- #elif defined(__SSE2__) || defined(__ARM_NEON) //----------------------------- SSE ---------------------------------------------------------------
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define VI16(ip, i, iv, parm)
-#define VI32(ip, i, iv, parm)
-#define IP16(_ip_, i, iv) _mm_loadu_si128(_ip_++)
-#define IP32(_ip_, i, iv) _mm_loadu_si128(_ip_++)
-#include "bitpack_.h"
-unsigned char *bitpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(128*b); BITPACK128V16(in, b, out, 0); return pout; }
-unsigned char *bitpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return pout; }
-unsigned char *bitpack256w32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *_out=out; unsigned *_in=in;
-BITPACK128V32(in, b, out, 0); in = _in+128; out = _out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return _out+PAD8(256*b); }
-
-#ifdef __ARM_NEON
-//#define IP32(_ip_, i, iv) _mm_or_si128(_mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(3, 1, 2, 0)), _mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(2, 0, 3, 1)) )
-#define IP32(_ip_, _i_, _iv_) _mm_or_si128(mm_shuffle_3120_epi32(_mm_loadu_si128(_ip_++) ), mm_shuffle_2031_epi32(_mm_loadu_si128(_ip++) ) ) // optimized shuffle
-#else
-#define IP32(_ip_, i, iv) _mm_or_si128(_mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(2, 0, 3, 1)), _mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(3, 1, 2, 0)) )
-#endif
-#include "bitpack_.h"
-unsigned char *bitpack128v64(uint64_t *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) {
- if(b<=32) { unsigned char *pout = out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return pout; } else return bitpack64(in,n,out,b);
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi16(v,_sv_); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi32(v,_sv_); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitdpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v,sv = _mm_set1_epi16(start); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitdpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v,sv = _mm_set1_epi32(start); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_)
-#define VI32(_ip_, _i_, _iv_, _sv_)
-#define IP16(_ip_, i, _iv_) _mm_sub_epi16(_mm_loadu_si128(_ip_++),sv)
-#define IP32(_ip_, i, _iv_) _mm_sub_epi32(_mm_loadu_si128(_ip_++),sv)
-#include "bitpack_.h"
-unsigned char *bitfpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitfpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi16(mm_delta_epi16(v,_sv_),cv); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi32(mm_delta_epi32(v,_sv_),cv); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitd1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(1), v; BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitd1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(1); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi16(SUBI16x8(v,_sv_),cv); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi32(SUBI32x4(v,_sv_),cv); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bits1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(8); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bits1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(4); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) _iv_ = _mm_sub_epi16(_mm_loadu_si128(_ip_++),_sv_); _sv_ = _mm_add_epi16(_sv_,cv);
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm_sub_epi32(_mm_loadu_si128(_ip_++),_sv_); _sv_ = _mm_add_epi32(_sv_,cv);
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitf1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set_epi16(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1), cv = _mm_set1_epi16(8); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitf1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set_epi32( start+4,start+3,start+2,start+1), cv = _mm_set1_epi32(4); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi16(v,_sv_); _sv_ = v; _iv_ = mm_zzage_epi16(_iv_)
-unsigned char *bitzpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(1); BITPACK128V16(in, b, out, sv); return pout;
-}
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi32(v,_sv_); _sv_ = v; _iv_ = mm_zzage_epi32(_iv_)
-unsigned char *bitzpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(1); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-size_t bitnpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip; _BITNPACKV( in, n, out, 128, 16, bitpack128v); }
-size_t bitnpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 128, 32, bitpack128v); }
-size_t bitnpack128v64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip; _BITNPACKV( in, n, out, 128, 64, bitpack128v); }
-size_t bitnpack256w32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 256, 32, bitpack256w); }
-
-size_t bitndpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitd, bitdpack128v, bitd, bitdpack); }
-size_t bitndpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitd, bitdpack128v, bitd, bitdpack); }
-
-size_t bitnd1pack128v16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitd1, bitd1pack128v, bitd1, bitd1pack); }
-size_t bitnd1pack128v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitd1, bitd1pack128v, bitd1, bitd1pack); }
-
-size_t bitns1pack128v16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bits128v, bits1pack128v, bitd1, bitd1pack); }
-size_t bitns1pack128v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bits128v, bits1pack128v, bitd1, bitd1pack); }
-
-size_t bitnzpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitz, bitzpack128v, bitz, bitzpack); }
-size_t bitnzpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitz, bitzpack128v, bitz, bitzpack); }
-
-size_t bitnfpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitf, bitfpack128v, bitf, bitfpack); }
-size_t bitnfpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitf, bitfpack128v, bitf, bitfpack); }
- #endif // SSE
-
-#pragma clang diagnostic pop
diff --git a/src/ext/for/bitpack.h b/src/ext/for/bitpack.h
deleted file mode 100644
index b8f91ad5690..00000000000
--- a/src/ext/for/bitpack.h
+++ /dev/null
@@ -1,327 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// bitpack.h - "Integer Compression" Binary Packing header file
-#ifndef BITPACK_H_
-#define BITPACK_H_
-#if defined(_MSC_VER) && _MSC_VER < 1600
-#include "vs/stdint.h"
-#else
-#include <stdint.h>
-#endif
-#include <stddef.h>
-#include <stdbool.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-//******************** Bit Packing High Level API - n unlimited ***************************************************
-size_t bitnpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitndpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnd1pack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnzpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnfpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnunpack128v64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitndunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitndunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitndunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitndunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitndunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitndunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitndunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnd1unpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnd1unpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnd1unpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnd1unpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnd1unpack128v16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnd1unpack128v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnd1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnzunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnzunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnzunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnzunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnzunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnzunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnzunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnfunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnfunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnfunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnfunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnfunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnfunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnfunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-//******** Bit Packing Low level API ****************************************************************
-// bipackNN: Pack array with n unsigned (NN bits in[n]) values to the buffer out using nbits per value. Return value = end of compressed buffer out
-unsigned char *bitpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-
-// delta bit packing
-unsigned char *bitdpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitdpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitdpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitdpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-unsigned char *bitd1pack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitd1pack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitd1pack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitd1pack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-// FOR bit packing : sorted integer array
-unsigned char *bitfpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitfpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitfpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitfpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-unsigned char *bitf1pack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitf1pack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitf1pack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitf1pack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-// zigzag : unsorted integer array
-unsigned char *bitzpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitzpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitzpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitzpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-//-------------------------------------- SIMD ------------------------------------------------------------------------------------------
-// Pack array with 128 unsigned (32 bits in[n]) values to the buffer out using nbits per value. Return value = end of compressed buffer out
-unsigned char *bitpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitd1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitfpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitf1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitzpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-
-unsigned char *bitpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitzpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-
-//unsigned char *bitpack256w32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack128v64( uint64_t *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-
-unsigned char *bitpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitzpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-
-//********************************** Bit Packing : Unpack ****************************************************************
-
-// ---------------- Unpack a b-bits packed integer array -------------------------------------------------------------------------------
-// unpack a bitpacked integer array. Return value = end of packed buffer in
-unsigned char *bitunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, unsigned b);
-unsigned char *bitunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, unsigned b);
-unsigned char *bitunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, unsigned b);
-unsigned char *bitunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b);
-
-// ---------------- Direct Access to a single packed integer array entry --------------------------------------------------------------
- #ifdef TURBOPFOR_DAC
- #ifdef __AVX2__
-#include <immintrin.h>
-#define bzhi64(_u_, _b_) _bzhi_u64(_u_, _b_)
-#define bzhi32(_u_, _b_) _bzhi_u32(_u_, _b_)
- #else
-#define bzhi64(_u_, _b_) ((_u_) & ((1ull<<(_b_))-1))
-#define bzhi32(_u_, _b_) ((_u_) & ((1u <<(_b_))-1))
- #endif
-
-#include "conf.h"
-
-static ALWAYS_INLINE unsigned bitgetx32(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-//static ALWAYS_INLINE unsigned bitgetx32(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx;
- //return (ctou64((uint32_t *)in+(bidx>>5)) << 32+(bidx&0x1f)) >> (64-b);
-// return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-static ALWAYS_INLINE unsigned _bitgetx32(const unsigned char *__restrict in, uint64_t bidx, unsigned b) { return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-
-// like bitgetx32 but for 16 bits integer array
-static ALWAYS_INLINE unsigned bitgetx8( const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi32( ctou16((uint16_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned _bitgetx8( const unsigned char *__restrict in, unsigned bidx, unsigned b) { return bzhi32( ctou16((uint16_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned bitgetx16(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi32( ctou32((uint32_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned _bitgetx16(const unsigned char *__restrict in, unsigned bidx, unsigned b) { return bzhi32( ctou32((uint32_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-
-// Set a single value with index "idx"
-static ALWAYS_INLINE void bitsetx16(const unsigned char *__restrict in, unsigned idx, unsigned v, unsigned b) { unsigned bidx = b*idx; unsigned *p = (unsigned *) in+(bidx>>4) ; *p = ( *p & ~(((1u <<b)-1) << (bidx& 0xf)) ) | v<<(bidx& 0xf);}
-static ALWAYS_INLINE void bitsetx32(const unsigned char *__restrict in, unsigned idx, unsigned v, unsigned b) { unsigned bidx = b*idx; unsigned long long *p = (unsigned long long *)((unsigned *)in+(bidx>>5)); *p = ( *p & ~(((1ull<<b)-1) << (bidx&0x1f)) ) | (unsigned long long)v<<(bidx&0x1f);}
- #endif
-// ---------------- DFOR : integrated bitpacking, for delta packed SORTED array (Ex. DocId in inverted index) -------------------------------
-// start <= out[0] <= out[1] <= ... <= out[n-2] <= out[n-1] <= (1<<N)-1 N=8,16,32 or 64
-// out[0] = start + in[0]; out[1] = out[0] + in[1]; ... ; out[i] = out[i-1] + in[i]
-unsigned char *bitdunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitdunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitdunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitdunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// start < out[0] < out[1] < ... < out[n-2] < out[n-1] < (1<<N)-1, N=8,16,32 or 64
-// out[0] = start + in[0] + 1; out[1] = out[0] + in[1] + 1; ... ; out[i] = out[i-1] + in[i] + 1
-unsigned char *bitd1unpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitd1unpack16(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitd1unpack32(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitd1unpack64(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- ZigZag : integrated bitpacking, for zigzag packed unsorted
-unsigned char *bitzunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitzunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitzunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitzunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- For : Direct Access for packed SORTED array --------------------------------------------
-// out[i] = start + in[i] + i
-unsigned char *bitfunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitfunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitfunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitfunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// out[i] = start + in[i] + i + 1
-unsigned char *bitf1unpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitf1unpack16(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitf1unpack32(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitf1unpack64(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- SIMD : unpack a SIMD bit packed integer array -------------------------------------------------------------------------------
-// SIMD unpack a 128/256 bitpacked integer array. Return value = end of packed buffer in
-unsigned char *bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b);
-unsigned char *bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitdunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitd1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitfunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitf1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-
-unsigned char *bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-unsigned char *bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b);
-
-unsigned char *bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitzunpack256scalarv32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-unsigned char *bitunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack128h32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-// internal TurboPFor functions: masked unpack
-unsigned char *_bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128h32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b, uint32_t *__restrict pex, unsigned char *bb);
-
-unsigned char* bitd1unpack256scalarv32(const unsigned char* __restrict in, unsigned n,
- unsigned* __restrict out, unsigned start, unsigned b);
-unsigned char* _bitd1unpack256scalarv32(const unsigned char* __restrict in, unsigned n,
- unsigned* __restrict out, unsigned start, unsigned b,
- unsigned* __restrict pex, unsigned char* bb);
-unsigned char *bitunpack256scalarv32(const unsigned char *__restrict in, unsigned n,
- unsigned *__restrict out, unsigned b);
-unsigned char *_bitunpack256scalarv32(const unsigned char *__restrict in,
- unsigned n,
- unsigned *__restrict out,
- unsigned b,
- unsigned *__restrict pex,
- unsigned char *bb,
- bool isZigZag);
-unsigned char *_bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack256scalarv32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-#ifdef __cplusplus
-}
-#endif
-#endif
-
diff --git a/src/ext/for/bitpack_.h b/src/ext/for/bitpack_.h
deleted file mode 100644
index d20cf9f0fea..00000000000
--- a/src/ext/for/bitpack_.h
+++ /dev/null
@@ -1,5190 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2017
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// TurboPFor Integer Compression: Bit Packing
-#ifdef IPI
-#define BITBLK64_1(ip, i, op, parm) { uint32_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint32_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint32_t)IPV(ip, i*32+ 1) << 1;\
- IP9(ip, i*32+ 2, parm); w |= (uint32_t)IPV(ip, i*32+ 2) << 2;\
- IP9(ip, i*32+ 3, parm); w |= (uint32_t)IPV(ip, i*32+ 3) << 3;\
- IP9(ip, i*32+ 4, parm); w |= (uint32_t)IPV(ip, i*32+ 4) << 4;\
- IP9(ip, i*32+ 5, parm); w |= (uint32_t)IPV(ip, i*32+ 5) << 5;\
- IP9(ip, i*32+ 6, parm); w |= (uint32_t)IPV(ip, i*32+ 6) << 6;\
- IP9(ip, i*32+ 7, parm); w |= (uint32_t)IPV(ip, i*32+ 7) << 7;\
- IP9(ip, i*32+ 8, parm); w |= (uint32_t)IPV(ip, i*32+ 8) << 8;\
- IP9(ip, i*32+ 9, parm); w |= (uint32_t)IPV(ip, i*32+ 9) << 9;\
- IP9(ip, i*32+10, parm); w |= (uint32_t)IPV(ip, i*32+10) << 10;\
- IP9(ip, i*32+11, parm); w |= (uint32_t)IPV(ip, i*32+11) << 11;\
- IP9(ip, i*32+12, parm); w |= (uint32_t)IPV(ip, i*32+12) << 12;\
- IP9(ip, i*32+13, parm); w |= (uint32_t)IPV(ip, i*32+13) << 13;\
- IP9(ip, i*32+14, parm); w |= (uint32_t)IPV(ip, i*32+14) << 14;\
- IP9(ip, i*32+15, parm); w |= (uint32_t)IPV(ip, i*32+15) << 15;\
- IP9(ip, i*32+16, parm); w |= (uint32_t)IPV(ip, i*32+16) << 16;\
- IP9(ip, i*32+17, parm); w |= (uint32_t)IPV(ip, i*32+17) << 17;\
- IP9(ip, i*32+18, parm); w |= (uint32_t)IPV(ip, i*32+18) << 18;\
- IP9(ip, i*32+19, parm); w |= (uint32_t)IPV(ip, i*32+19) << 19;\
- IP9(ip, i*32+20, parm); w |= (uint32_t)IPV(ip, i*32+20) << 20;\
- IP9(ip, i*32+21, parm); w |= (uint32_t)IPV(ip, i*32+21) << 21;\
- IP9(ip, i*32+22, parm); w |= (uint32_t)IPV(ip, i*32+22) << 22;\
- IP9(ip, i*32+23, parm); w |= (uint32_t)IPV(ip, i*32+23) << 23;\
- IP9(ip, i*32+24, parm); w |= (uint32_t)IPV(ip, i*32+24) << 24;\
- IP9(ip, i*32+25, parm); w |= (uint32_t)IPV(ip, i*32+25) << 25;\
- IP9(ip, i*32+26, parm); w |= (uint32_t)IPV(ip, i*32+26) << 26;\
- IP9(ip, i*32+27, parm); w |= (uint32_t)IPV(ip, i*32+27) << 27;\
- IP9(ip, i*32+28, parm); w |= (uint32_t)IPV(ip, i*32+28) << 28;\
- IP9(ip, i*32+29, parm); w |= (uint32_t)IPV(ip, i*32+29) << 29;\
- IP9(ip, i*32+30, parm); w |= (uint32_t)IPV(ip, i*32+30) << 30;\
- IP9(ip, i*32+31, parm); w |= (uint32_t)IPV(ip, i*32+31) << 31;*((uint32_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_1(ip, op, parm) { \
- BITBLK64_1(ip, 0, op, parm); IPI(ip); op += 1*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_2(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint64_t)IPV(ip, i*32+ 1) << 2;\
- IP9(ip, i*32+ 2, parm); w |= (uint64_t)IPV(ip, i*32+ 2) << 4;\
- IP9(ip, i*32+ 3, parm); w |= (uint64_t)IPV(ip, i*32+ 3) << 6;\
- IP9(ip, i*32+ 4, parm); w |= (uint64_t)IPV(ip, i*32+ 4) << 8;\
- IP9(ip, i*32+ 5, parm); w |= (uint64_t)IPV(ip, i*32+ 5) << 10;\
- IP9(ip, i*32+ 6, parm); w |= (uint64_t)IPV(ip, i*32+ 6) << 12;\
- IP9(ip, i*32+ 7, parm); w |= (uint64_t)IPV(ip, i*32+ 7) << 14;\
- IP9(ip, i*32+ 8, parm); w |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); w |= (uint64_t)IPV(ip, i*32+ 9) << 18;\
- IP9(ip, i*32+10, parm); w |= (uint64_t)IPV(ip, i*32+10) << 20;\
- IP9(ip, i*32+11, parm); w |= (uint64_t)IPV(ip, i*32+11) << 22;\
- IP9(ip, i*32+12, parm); w |= (uint64_t)IPV(ip, i*32+12) << 24;\
- IP9(ip, i*32+13, parm); w |= (uint64_t)IPV(ip, i*32+13) << 26;\
- IP9(ip, i*32+14, parm); w |= (uint64_t)IPV(ip, i*32+14) << 28;\
- IP9(ip, i*32+15, parm); w |= (uint64_t)IPV(ip, i*32+15) << 30;\
- IP9(ip, i*32+16, parm); w |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); w |= (uint64_t)IPV(ip, i*32+17) << 34;\
- IP9(ip, i*32+18, parm); w |= (uint64_t)IPV(ip, i*32+18) << 36;\
- IP9(ip, i*32+19, parm); w |= (uint64_t)IPV(ip, i*32+19) << 38;\
- IP9(ip, i*32+20, parm); w |= (uint64_t)IPV(ip, i*32+20) << 40;\
- IP9(ip, i*32+21, parm); w |= (uint64_t)IPV(ip, i*32+21) << 42;\
- IP9(ip, i*32+22, parm); w |= (uint64_t)IPV(ip, i*32+22) << 44;\
- IP9(ip, i*32+23, parm); w |= (uint64_t)IPV(ip, i*32+23) << 46;\
- IP9(ip, i*32+24, parm); w |= (uint64_t)IPV(ip, i*32+24) << 48;\
- IP9(ip, i*32+25, parm); w |= (uint64_t)IPV(ip, i*32+25) << 50;\
- IP9(ip, i*32+26, parm); w |= (uint64_t)IPV(ip, i*32+26) << 52;\
- IP9(ip, i*32+27, parm); w |= (uint64_t)IPV(ip, i*32+27) << 54;\
- IP9(ip, i*32+28, parm); w |= (uint64_t)IPV(ip, i*32+28) << 56;\
- IP9(ip, i*32+29, parm); w |= (uint64_t)IPV(ip, i*32+29) << 58;\
- IP9(ip, i*32+30, parm); w |= (uint64_t)IPV(ip, i*32+30) << 60;\
- IP9(ip, i*32+31, parm); w |= (uint64_t)IPV(ip, i*32+31) << 62;*((uint64_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_2(ip, op, parm) { \
- BITBLK64_2(ip, 0, op, parm); IPI(ip); op += 2*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_3(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 3;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 6;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 9;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 12;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 15;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 18;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 21;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); w |= (uint64_t)IPV(ip, i*64+ 9) << 27;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 30;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 33;\
- IP9(ip, i*64+12, parm); w |= (uint64_t)IPV(ip, i*64+12) << 36;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 39;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 42;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 45;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 48;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 51;\
- IP9(ip, i*64+18, parm); w |= (uint64_t)IPV(ip, i*64+18) << 54;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 57;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 60 | (uint64_t)IPX(ip, i*64+21) << 63;*((uint64_t *)op+i*3+ 0) = w;\
- IP64(ip, i*64+21, parm); w = (uint64_t)IPW(ip, i*64+21) >> 1;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 2;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 5;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); w |= (uint64_t)IPV(ip, i*64+25) << 11;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 14;\
- IP9(ip, i*64+27, parm); w |= (uint64_t)IPV(ip, i*64+27) << 17;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 20;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 23;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 26;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 29;*((uint64_t *)op+i*3+ 1) = w;;\
-}
-
-#define BITPACK64_3(ip, op, parm) { \
- BITBLK64_3(ip, 0, op, parm); IPI(ip); op += 3*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_4(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*16+ 0, parm); w = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); w |= (uint64_t)IPV(ip, i*16+ 1) << 4;\
- IP9(ip, i*16+ 2, parm); w |= (uint64_t)IPV(ip, i*16+ 2) << 8;\
- IP9(ip, i*16+ 3, parm); w |= (uint64_t)IPV(ip, i*16+ 3) << 12;\
- IP9(ip, i*16+ 4, parm); w |= (uint64_t)IPV(ip, i*16+ 4) << 16;\
- IP9(ip, i*16+ 5, parm); w |= (uint64_t)IPV(ip, i*16+ 5) << 20;\
- IP9(ip, i*16+ 6, parm); w |= (uint64_t)IPV(ip, i*16+ 6) << 24;\
- IP9(ip, i*16+ 7, parm); w |= (uint64_t)IPV(ip, i*16+ 7) << 28;\
- IP9(ip, i*16+ 8, parm); w |= (uint64_t)IPV(ip, i*16+ 8) << 32;\
- IP9(ip, i*16+ 9, parm); w |= (uint64_t)IPV(ip, i*16+ 9) << 36;\
- IP9(ip, i*16+10, parm); w |= (uint64_t)IPV(ip, i*16+10) << 40;\
- IP9(ip, i*16+11, parm); w |= (uint64_t)IPV(ip, i*16+11) << 44;\
- IP9(ip, i*16+12, parm); w |= (uint64_t)IPV(ip, i*16+12) << 48;\
- IP9(ip, i*16+13, parm); w |= (uint64_t)IPV(ip, i*16+13) << 52;\
- IP9(ip, i*16+14, parm); w |= (uint64_t)IPV(ip, i*16+14) << 56;\
- IP9(ip, i*16+15, parm); w |= (uint64_t)IPV(ip, i*16+15) << 60;*((uint64_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_4(ip, op, parm) { \
- BITBLK64_4(ip, 0, op, parm);\
- BITBLK64_4(ip, 1, op, parm); IPI(ip); op += 4*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_5(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 5;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 10;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 15;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 20;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 25;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 30;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 35;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 40;\
- IP9(ip, i*64+ 9, parm); w |= (uint64_t)IPV(ip, i*64+ 9) << 45;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 50;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 55 | (uint64_t)IPX(ip, i*64+12) << 60;*((uint64_t *)op+i*5+ 0) = w;\
- IP64(ip, i*64+12, parm); w = (uint64_t)IPW(ip, i*64+12) >> 4;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 1;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 6;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 11;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 21;\
- IP9(ip, i*64+18, parm); w |= (uint64_t)IPV(ip, i*64+18) << 26;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 31;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 36;\
- IP9(ip, i*64+21, parm); w |= (uint64_t)IPV(ip, i*64+21) << 41;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 46;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 51;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 56 | (uint64_t)IPX(ip, i*64+25) << 61;*((uint64_t *)op+i*5+ 1) = w;\
- IP64(ip, i*64+25, parm); w = (uint64_t)IPW(ip, i*64+25) >> 3;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 2;\
- IP9(ip, i*64+27, parm); w |= (uint64_t)IPV(ip, i*64+27) << 7;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 12;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 17;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 22;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 27;*((uint64_t *)op+i*5+ 2) = w;;\
-}
-
-#define BITPACK64_5(ip, op, parm) { \
- BITBLK64_5(ip, 0, op, parm); IPI(ip); op += 5*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_6(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint64_t)IPV(ip, i*32+ 1) << 6;\
- IP9(ip, i*32+ 2, parm); w |= (uint64_t)IPV(ip, i*32+ 2) << 12;\
- IP9(ip, i*32+ 3, parm); w |= (uint64_t)IPV(ip, i*32+ 3) << 18;\
- IP9(ip, i*32+ 4, parm); w |= (uint64_t)IPV(ip, i*32+ 4) << 24;\
- IP9(ip, i*32+ 5, parm); w |= (uint64_t)IPV(ip, i*32+ 5) << 30;\
- IP9(ip, i*32+ 6, parm); w |= (uint64_t)IPV(ip, i*32+ 6) << 36;\
- IP9(ip, i*32+ 7, parm); w |= (uint64_t)IPV(ip, i*32+ 7) << 42;\
- IP9(ip, i*32+ 8, parm); w |= (uint64_t)IPV(ip, i*32+ 8) << 48;\
- IP9(ip, i*32+ 9, parm); w |= (uint64_t)IPV(ip, i*32+ 9) << 54 | (uint64_t)IPX(ip, i*32+10) << 60;*((uint64_t *)op+i*3+ 0) = w;\
- IP64(ip, i*32+10, parm); w = (uint64_t)IPW(ip, i*32+10) >> 4;\
- IP9(ip, i*32+11, parm); w |= (uint64_t)IPV(ip, i*32+11) << 2;\
- IP9(ip, i*32+12, parm); w |= (uint64_t)IPV(ip, i*32+12) << 8;\
- IP9(ip, i*32+13, parm); w |= (uint64_t)IPV(ip, i*32+13) << 14;\
- IP9(ip, i*32+14, parm); w |= (uint64_t)IPV(ip, i*32+14) << 20;\
- IP9(ip, i*32+15, parm); w |= (uint64_t)IPV(ip, i*32+15) << 26;\
- IP9(ip, i*32+16, parm); w |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); w |= (uint64_t)IPV(ip, i*32+17) << 38;\
- IP9(ip, i*32+18, parm); w |= (uint64_t)IPV(ip, i*32+18) << 44;\
- IP9(ip, i*32+19, parm); w |= (uint64_t)IPV(ip, i*32+19) << 50;\
- IP9(ip, i*32+20, parm); w |= (uint64_t)IPV(ip, i*32+20) << 56 | (uint64_t)IPX(ip, i*32+21) << 62;*((uint64_t *)op+i*3+ 1) = w;\
- IP64(ip, i*32+21, parm); w = (uint64_t)IPW(ip, i*32+21) >> 2;\
- IP9(ip, i*32+22, parm); w |= (uint64_t)IPV(ip, i*32+22) << 4;\
- IP9(ip, i*32+23, parm); w |= (uint64_t)IPV(ip, i*32+23) << 10;\
- IP9(ip, i*32+24, parm); w |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); w |= (uint64_t)IPV(ip, i*32+25) << 22;\
- IP9(ip, i*32+26, parm); w |= (uint64_t)IPV(ip, i*32+26) << 28;\
- IP9(ip, i*32+27, parm); w |= (uint64_t)IPV(ip, i*32+27) << 34;\
- IP9(ip, i*32+28, parm); w |= (uint64_t)IPV(ip, i*32+28) << 40;\
- IP9(ip, i*32+29, parm); w |= (uint64_t)IPV(ip, i*32+29) << 46;\
- IP9(ip, i*32+30, parm); w |= (uint64_t)IPV(ip, i*32+30) << 52;\
- IP9(ip, i*32+31, parm); w |= (uint64_t)IPV(ip, i*32+31) << 58;*((uint64_t *)op+i*3+ 2) = w;;\
-}
-
-#define BITPACK64_6(ip, op, parm) { \
- BITBLK64_6(ip, 0, op, parm); IPI(ip); op += 6*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_7(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 7;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 14;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 21;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 28;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 35;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 42;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 49;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 56 | (uint64_t)IPX(ip, i*64+9) << 63;*((uint64_t *)op+i*7+ 0) = w;\
- IP64(ip, i*64+ 9, parm); w = (uint64_t)IPW(ip, i*64+ 9) >> 1;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 6;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 13;\
- IP9(ip, i*64+12, parm); w |= (uint64_t)IPV(ip, i*64+12) << 20;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 27;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 34;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 41;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 48;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 55 | (uint64_t)IPX(ip, i*64+18) << 62;*((uint64_t *)op+i*7+ 1) = w;\
- IP64(ip, i*64+18, parm); w = (uint64_t)IPW(ip, i*64+18) >> 2;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 5;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 12;\
- IP9(ip, i*64+21, parm); w |= (uint64_t)IPV(ip, i*64+21) << 19;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 26;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 33;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 40;\
- IP9(ip, i*64+25, parm); w |= (uint64_t)IPV(ip, i*64+25) << 47;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 54 | (uint64_t)IPX(ip, i*64+27) << 61;*((uint64_t *)op+i*7+ 2) = w;\
- IP64(ip, i*64+27, parm); w = (uint64_t)IPW(ip, i*64+27) >> 3;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 4;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 11;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 18;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 25;*((uint64_t *)op+i*7+ 3) = w;;\
-}
-
-#define BITPACK64_7(ip, op, parm) { \
- BITBLK64_7(ip, 0, op, parm); IPI(ip); op += 7*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_8(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*1+ 0) = (uint64_t)IPV(ip, i*8+ 0) ;\
- IP9(ip, i*8+ 1, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 1) << 8;\
- IP9(ip, i*8+ 2, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 2) << 16;\
- IP9(ip, i*8+ 3, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 3) << 24;\
- IP9(ip, i*8+ 4, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 4) << 32;\
- IP9(ip, i*8+ 5, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 5) << 40;\
- IP9(ip, i*8+ 6, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 6) << 48;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 7) << 56;\
-}
-
-#define BITPACK64_8(ip, op, parm) { \
- BITBLK64_8(ip, 0, op, parm);\
- BITBLK64_8(ip, 1, op, parm);\
- BITBLK64_8(ip, 2, op, parm);\
- BITBLK64_8(ip, 3, op, parm); IPI(ip); op += 8*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_9(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 9;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 18;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 27;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 4) << 36;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 5) << 45;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 6) << 54 | (uint64_t)IPX(ip, i*64+7) << 63;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*64+ 7) >> 1;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+ 9) << 17;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+10) << 26;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+11) << 35;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+12) << 44;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+13) << 53 | (uint64_t)IPX(ip, i*64+14) << 62;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*64+14) >> 2;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+15) << 7;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+17) << 25;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+18) << 34;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+19) << 43;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+20) << 52 | (uint64_t)IPX(ip, i*64+21) << 61;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*64+21) >> 3;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+22) << 6;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+23) << 15;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+24) << 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+25) << 33;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+26) << 42;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+27) << 51 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*64+28) >> 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+29) << 5;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+30) << 14;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+31) << 23;\
-}
-
-#define BITPACK64_9(ip, op, parm) { \
- BITBLK64_9(ip, 0, op, parm); IPI(ip); op += 9*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_10(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 10;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 20;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 3) << 30;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 4) << 40;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 5) << 50 | (uint64_t)IPX(ip, i*32+6) << 60;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*32+ 6) >> 4;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 7) << 6;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 9) << 26;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+10) << 36;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+11) << 46 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*32+12) >> 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+13) << 2;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+14) << 12;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+15) << 22;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+17) << 42;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+18) << 52 | (uint64_t)IPX(ip, i*32+19) << 62;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*32+19) >> 2;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+20) << 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+21) << 18;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+22) << 28;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+23) << 38;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+24) << 48 | (uint64_t)IPX(ip, i*32+25) << 58;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*32+25) >> 6;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+26) << 4;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+27) << 14;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+28) << 24;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+29) << 34;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+30) << 44;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+31) << 54;\
-}
-
-#define BITPACK64_10(ip, op, parm) { \
- BITBLK64_10(ip, 0, op, parm); IPI(ip); op += 10*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_11(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 11;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 22;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 33;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 4) << 44 | (uint64_t)IPX(ip, i*64+5) << 55;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*64+ 5) >> 9;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 2;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 13;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 9) << 35;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+10) << 46 | (uint64_t)IPX(ip, i*64+11) << 57;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*64+11) >> 7;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+12) << 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+13) << 15;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+14) << 26;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+15) << 37;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+16) << 48 | (uint64_t)IPX(ip, i*64+17) << 59;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*64+17) >> 5;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+18) << 6;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+19) << 17;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+20) << 28;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+21) << 39;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+22) << 50 | (uint64_t)IPX(ip, i*64+23) << 61;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*64+23) >> 3;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+25) << 19;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+26) << 30;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+27) << 41;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+28) << 52 | (uint64_t)IPX(ip, i*64+29) << 63;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*64+29) >> 1;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*64+30) << 10;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*64+31) << 21;\
-}
-
-#define BITPACK64_11(ip, op, parm) { \
- BITBLK64_11(ip, 0, op, parm); IPI(ip); op += 11*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_12(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 12;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 2) << 24;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 3) << 36;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 4) << 48 | (uint64_t)IPX(ip, i*16+5) << 60;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*16+ 5) >> 4;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 6) << 8;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 7) << 20;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 8) << 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 9) << 44 | (uint64_t)IPX(ip, i*16+10) << 56;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*16+10) >> 8;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+11) << 4;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+12) << 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+13) << 28;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+14) << 40;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+15) << 52;\
-}
-
-#define BITPACK64_12(ip, op, parm) { \
- BITBLK64_12(ip, 0, op, parm);\
- BITBLK64_12(ip, 1, op, parm); IPI(ip); op += 12*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_13(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 13;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 26;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 39 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*64+ 4) >> 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 1;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 14;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 27;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 40 | (uint64_t)IPX(ip, i*64+9) << 53;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*64+ 9) >> 11;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+10) << 2;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+11) << 15;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+12) << 28;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+13) << 41 | (uint64_t)IPX(ip, i*64+14) << 54;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*64+14) >> 10;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+15) << 3;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+17) << 29;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+18) << 42 | (uint64_t)IPX(ip, i*64+19) << 55;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*64+19) >> 9;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+20) << 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+21) << 17;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+22) << 30;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+23) << 43 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+25) << 5;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+26) << 18;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+27) << 31;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+28) << 44 | (uint64_t)IPX(ip, i*64+29) << 57;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*64+29) >> 7;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*64+30) << 6;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*64+31) << 19;\
-}
-
-#define BITPACK64_13(ip, op, parm) { \
- BITBLK64_13(ip, 0, op, parm); IPI(ip); op += 13*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_14(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 14;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 28;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 3) << 42 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*32+ 4) >> 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 5) << 6;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 6) << 20;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 7) << 34;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 8) << 48 | (uint64_t)IPX(ip, i*32+9) << 62;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*32+ 9) >> 2;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+10) << 12;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+11) << 26;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+12) << 40 | (uint64_t)IPX(ip, i*32+13) << 54;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*32+13) >> 10;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+14) << 4;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+15) << 18;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+17) << 46 | (uint64_t)IPX(ip, i*32+18) << 60;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*32+18) >> 4;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+19) << 10;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+20) << 24;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+21) << 38 | (uint64_t)IPX(ip, i*32+22) << 52;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*32+22) >> 12;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+23) << 2;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+25) << 30;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+26) << 44 | (uint64_t)IPX(ip, i*32+27) << 58;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*32+27) >> 6;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+28) << 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+29) << 22;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+30) << 36;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+31) << 50;\
-}
-
-#define BITPACK64_14(ip, op, parm) { \
- BITBLK64_14(ip, 0, op, parm); IPI(ip); op += 14*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_15(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 15;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 30;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 45 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*64+ 4) >> 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 11;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 26;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 41 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 7;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+10) << 22;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+11) << 37 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*64+12) >> 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+13) << 3;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+14) << 18;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+15) << 33;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+16) << 48 | (uint64_t)IPX(ip, i*64+17) << 63;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*64+17) >> 1;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+18) << 14;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+19) << 29;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+20) << 44 | (uint64_t)IPX(ip, i*64+21) << 59;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*64+21) >> 5;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+22) << 10;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+23) << 25;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+24) << 40 | (uint64_t)IPX(ip, i*64+25) << 55;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*64+25) >> 9;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+26) << 6;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+27) << 21;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+28) << 36 | (uint64_t)IPX(ip, i*64+29) << 51;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*64+29) >> 13;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*64+30) << 2;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*64+31) << 17;\
-}
-
-#define BITPACK64_15(ip, op, parm) { \
- BITBLK64_15(ip, 0, op, parm); IPI(ip); op += 15*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_16(ip, i, op, parm) { \
- IP9(ip, i*4+ 0, parm); *(uint16_t *)(op+i*8+ 0) = IPV(ip, i*4+ 0);\
- IP9(ip, i*4+ 1, parm); *(uint16_t *)(op+i*8+ 2) = IPV(ip, i*4+ 1);\
- IP9(ip, i*4+ 2, parm); *(uint16_t *)(op+i*8+ 4) = IPV(ip, i*4+ 2);\
- IP9(ip, i*4+ 3, parm); *(uint16_t *)(op+i*8+ 6) = IPV(ip, i*4+ 3);;\
-}
-
-#define BITPACK64_16(ip, op, parm) { \
- BITBLK64_16(ip, 0, op, parm);\
- BITBLK64_16(ip, 1, op, parm);\
- BITBLK64_16(ip, 2, op, parm);\
- BITBLK64_16(ip, 3, op, parm);\
- BITBLK64_16(ip, 4, op, parm);\
- BITBLK64_16(ip, 5, op, parm);\
- BITBLK64_16(ip, 6, op, parm);\
- BITBLK64_16(ip, 7, op, parm); IPI(ip); op += 16*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_17(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*17+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*17+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 17;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*17+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 34 | (uint64_t)IPX(ip, i*64+3) << 51;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*17+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 13;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 21;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 38 | (uint64_t)IPX(ip, i*64+7) << 55;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*17+ 2) = (uint64_t)IPW(ip, i*64+ 7) >> 9;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 25;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+10) << 42 | (uint64_t)IPX(ip, i*64+11) << 59;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*17+ 3) = (uint64_t)IPW(ip, i*64+11) >> 5;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+12) << 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+13) << 29;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+14) << 46 | (uint64_t)IPX(ip, i*64+15) << 63;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*17+ 4) = (uint64_t)IPW(ip, i*64+15) >> 1;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*64+17) << 33 | (uint64_t)IPX(ip, i*64+18) << 50;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*17+ 5) = (uint64_t)IPW(ip, i*64+18) >> 14;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+19) << 3;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+20) << 20;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+21) << 37 | (uint64_t)IPX(ip, i*64+22) << 54;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*17+ 6) = (uint64_t)IPW(ip, i*64+22) >> 10;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+23) << 7;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+24) << 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+25) << 41 | (uint64_t)IPX(ip, i*64+26) << 58;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*17+ 7) = (uint64_t)IPW(ip, i*64+26) >> 6;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+27) << 11;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+28) << 28;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+29) << 45 | (uint64_t)IPX(ip, i*64+30) << 62;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*17+ 8) = (uint64_t)IPW(ip, i*64+30) >> 2;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*17+ 8) |= (uint64_t)IPV(ip, i*64+31) << 15;\
-}
-
-#define BITPACK64_17(ip, op, parm) { \
- BITBLK64_17(ip, 0, op, parm); IPI(ip); op += 17*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_18(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 18;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 36 | (uint64_t)IPX(ip, i*32+3) << 54;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*32+ 3) >> 10;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 4) << 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 5) << 26;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 6) << 44 | (uint64_t)IPX(ip, i*32+7) << 62;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*32+ 7) >> 2;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*32+ 9) << 34 | (uint64_t)IPX(ip, i*32+10) << 52;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*32+10) >> 12;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+11) << 6;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+12) << 24;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+13) << 42 | (uint64_t)IPX(ip, i*32+14) << 60;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*32+14) >> 4;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*32+15) << 14;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 50;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*9+ 5) = (uint64_t)IPW(ip, i*32+17) >> 14;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+18) << 4;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+19) << 22;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+20) << 40 | (uint64_t)IPX(ip, i*32+21) << 58;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*9+ 6) = (uint64_t)IPW(ip, i*32+21) >> 6;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*32+22) << 12;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*32+23) << 30 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*9+ 7) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+25) << 2;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+26) << 20;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+27) << 38 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*9+ 8) = (uint64_t)IPW(ip, i*32+28) >> 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+29) << 10;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+30) << 28;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+31) << 46;\
-}
-
-#define BITPACK64_18(ip, op, parm) { \
- BITBLK64_18(ip, 0, op, parm); IPI(ip); op += 18*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_19(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*19+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*19+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 19;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*19+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 38 | (uint64_t)IPX(ip, i*64+3) << 57;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*19+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 7;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 31 | (uint64_t)IPX(ip, i*64+6) << 50;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*19+ 2) = (uint64_t)IPW(ip, i*64+ 6) >> 14;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 5;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 43 | (uint64_t)IPX(ip, i*64+10) << 62;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*19+ 3) = (uint64_t)IPW(ip, i*64+10) >> 2;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*19+ 3) |= (uint64_t)IPV(ip, i*64+11) << 17;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*19+ 3) |= (uint64_t)IPV(ip, i*64+12) << 36 | (uint64_t)IPX(ip, i*64+13) << 55;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*19+ 4) = (uint64_t)IPW(ip, i*64+13) >> 9;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*64+14) << 10;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*64+15) << 29 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*19+ 5) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+17) << 3;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+18) << 22;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+19) << 41 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*19+ 6) = (uint64_t)IPW(ip, i*64+20) >> 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*19+ 6) |= (uint64_t)IPV(ip, i*64+21) << 15;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*19+ 6) |= (uint64_t)IPV(ip, i*64+22) << 34 | (uint64_t)IPX(ip, i*64+23) << 53;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*19+ 7) = (uint64_t)IPW(ip, i*64+23) >> 11;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*64+25) << 27 | (uint64_t)IPX(ip, i*64+26) << 46;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*19+ 8) = (uint64_t)IPW(ip, i*64+26) >> 18;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+27) << 1;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+28) << 20;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+29) << 39 | (uint64_t)IPX(ip, i*64+30) << 58;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*19+ 9) = (uint64_t)IPW(ip, i*64+30) >> 6;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*19+ 9) |= (uint64_t)IPV(ip, i*64+31) << 13;\
-}
-
-#define BITPACK64_19(ip, op, parm) { \
- BITBLK64_19(ip, 0, op, parm); IPI(ip); op += 19*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_20(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 20;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*16+ 2) << 40 | (uint64_t)IPX(ip, i*16+3) << 60;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*16+ 3) >> 4;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*16+ 4) << 16;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*16+ 5) << 36 | (uint64_t)IPX(ip, i*16+6) << 56;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*16+ 6) >> 8;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*16+ 7) << 12;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*16+ 8) << 32 | (uint64_t)IPX(ip, i*16+9) << 52;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*16+ 9) >> 12;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*16+10) << 8;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*16+11) << 28 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*16+12) >> 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+13) << 4;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+14) << 24;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+15) << 44;\
-}
-
-#define BITPACK64_20(ip, op, parm) { \
- BITBLK64_20(ip, 0, op, parm);\
- BITBLK64_20(ip, 1, op, parm); IPI(ip); op += 20*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_21(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*21+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*21+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 21;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*21+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 42 | (uint64_t)IPX(ip, i*64+3) << 63;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*21+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 1;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 20;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 41 | (uint64_t)IPX(ip, i*64+6) << 62;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*21+ 2) = (uint64_t)IPW(ip, i*64+ 6) >> 2;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*21+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 19;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*21+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 40 | (uint64_t)IPX(ip, i*64+9) << 61;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*21+ 3) = (uint64_t)IPW(ip, i*64+ 9) >> 3;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*64+10) << 18;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*64+11) << 39 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*21+ 4) = (uint64_t)IPW(ip, i*64+12) >> 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*21+ 4) |= (uint64_t)IPV(ip, i*64+13) << 17;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*21+ 4) |= (uint64_t)IPV(ip, i*64+14) << 38 | (uint64_t)IPX(ip, i*64+15) << 59;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*21+ 5) = (uint64_t)IPW(ip, i*64+15) >> 5;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*64+17) << 37 | (uint64_t)IPX(ip, i*64+18) << 58;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*21+ 6) = (uint64_t)IPW(ip, i*64+18) >> 6;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*21+ 6) |= (uint64_t)IPV(ip, i*64+19) << 15;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*21+ 6) |= (uint64_t)IPV(ip, i*64+20) << 36 | (uint64_t)IPX(ip, i*64+21) << 57;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*21+ 7) = (uint64_t)IPW(ip, i*64+21) >> 7;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*64+22) << 14;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*64+23) << 35 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*21+ 8) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*21+ 8) |= (uint64_t)IPV(ip, i*64+25) << 13;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*21+ 8) |= (uint64_t)IPV(ip, i*64+26) << 34 | (uint64_t)IPX(ip, i*64+27) << 55;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*21+ 9) = (uint64_t)IPW(ip, i*64+27) >> 9;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*64+28) << 12;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*64+29) << 33 | (uint64_t)IPX(ip, i*64+30) << 54;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*21+10) = (uint64_t)IPW(ip, i*64+30) >> 10;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*21+10) |= (uint64_t)IPV(ip, i*64+31) << 11;\
-}
-
-#define BITPACK64_21(ip, op, parm) { \
- BITBLK64_21(ip, 0, op, parm); IPI(ip); op += 21*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_22(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 22 | (uint64_t)IPX(ip, i*32+2) << 44;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 20;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 2;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*32+ 4) << 24 | (uint64_t)IPX(ip, i*32+5) << 46;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*32+ 5) >> 18;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*32+ 6) << 4;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*32+ 7) << 26 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*32+ 9) << 6;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*32+10) << 28 | (uint64_t)IPX(ip, i*32+11) << 50;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*32+11) >> 14;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*32+12) << 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*32+13) << 30 | (uint64_t)IPX(ip, i*32+14) << 52;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*32+14) >> 12;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*32+15) << 10;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 54;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*11+ 6) = (uint64_t)IPW(ip, i*32+17) >> 10;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*32+18) << 12;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*32+19) << 34 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*11+ 7) = (uint64_t)IPW(ip, i*32+20) >> 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*11+ 7) |= (uint64_t)IPV(ip, i*32+21) << 14;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*11+ 7) |= (uint64_t)IPV(ip, i*32+22) << 36 | (uint64_t)IPX(ip, i*32+23) << 58;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*11+ 8) = (uint64_t)IPW(ip, i*32+23) >> 6;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*32+25) << 38 | (uint64_t)IPX(ip, i*32+26) << 60;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*11+ 9) = (uint64_t)IPW(ip, i*32+26) >> 4;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*11+ 9) |= (uint64_t)IPV(ip, i*32+27) << 18;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*11+ 9) |= (uint64_t)IPV(ip, i*32+28) << 40 | (uint64_t)IPX(ip, i*32+29) << 62;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*11+10) = (uint64_t)IPW(ip, i*32+29) >> 2;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*32+30) << 20;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*32+31) << 42;\
-}
-
-#define BITPACK64_22(ip, op, parm) { \
- BITBLK64_22(ip, 0, op, parm); IPI(ip); op += 22*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_23(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*23+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*23+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 23 | (uint64_t)IPX(ip, i*64+2) << 46;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*23+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 18;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*23+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 5;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*23+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 28 | (uint64_t)IPX(ip, i*64+5) << 51;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*23+ 2) = (uint64_t)IPW(ip, i*64+ 5) >> 13;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 10;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 33 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*23+ 3) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*23+ 3) |= (uint64_t)IPV(ip, i*64+ 9) << 15;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*23+ 3) |= (uint64_t)IPV(ip, i*64+10) << 38 | (uint64_t)IPX(ip, i*64+11) << 61;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*23+ 4) = (uint64_t)IPW(ip, i*64+11) >> 3;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*23+ 4) |= (uint64_t)IPV(ip, i*64+12) << 20 | (uint64_t)IPX(ip, i*64+13) << 43;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*23+ 5) = (uint64_t)IPW(ip, i*64+13) >> 21;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*64+14) << 2;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*64+15) << 25 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*23+ 6) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*23+ 6) |= (uint64_t)IPV(ip, i*64+17) << 7;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*23+ 6) |= (uint64_t)IPV(ip, i*64+18) << 30 | (uint64_t)IPX(ip, i*64+19) << 53;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*23+ 7) = (uint64_t)IPW(ip, i*64+19) >> 11;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*64+20) << 12;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*64+21) << 35 | (uint64_t)IPX(ip, i*64+22) << 58;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*23+ 8) = (uint64_t)IPW(ip, i*64+22) >> 6;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*23+ 8) |= (uint64_t)IPV(ip, i*64+23) << 17;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*23+ 8) |= (uint64_t)IPV(ip, i*64+24) << 40 | (uint64_t)IPX(ip, i*64+25) << 63;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*23+ 9) = (uint64_t)IPW(ip, i*64+25) >> 1;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*23+ 9) |= (uint64_t)IPV(ip, i*64+26) << 22 | (uint64_t)IPX(ip, i*64+27) << 45;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*23+10) = (uint64_t)IPW(ip, i*64+27) >> 19;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*64+28) << 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*64+29) << 27 | (uint64_t)IPX(ip, i*64+30) << 50;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*23+11) = (uint64_t)IPW(ip, i*64+30) >> 14;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*23+11) |= (uint64_t)IPV(ip, i*64+31) << 9;\
-}
-
-#define BITPACK64_23(ip, op, parm) { \
- BITBLK64_23(ip, 0, op, parm); IPI(ip); op += 23*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_24(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*8+ 0) ;\
- IP9(ip, i*8+ 1, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*8+ 1) << 24 | (uint64_t)IPX(ip, i*8+2) << 48;\
- IP64(ip, i*8+ 2, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*8+ 2) >> 16;\
- IP9(ip, i*8+ 3, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*8+ 3) << 8;\
- IP9(ip, i*8+ 4, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*8+ 4) << 32 | (uint64_t)IPX(ip, i*8+5) << 56;\
- IP64(ip, i*8+ 5, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*8+ 5) >> 8;\
- IP9(ip, i*8+ 6, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*8+ 6) << 16;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*8+ 7) << 40;\
-}
-
-#define BITPACK64_24(ip, op, parm) { \
- BITBLK64_24(ip, 0, op, parm);\
- BITBLK64_24(ip, 1, op, parm);\
- BITBLK64_24(ip, 2, op, parm);\
- BITBLK64_24(ip, 3, op, parm); IPI(ip); op += 24*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_25(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*25+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*25+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 25 | (uint64_t)IPX(ip, i*64+2) << 50;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*25+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 14;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*25+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 11;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*25+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 36 | (uint64_t)IPX(ip, i*64+5) << 61;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*25+ 2) = (uint64_t)IPW(ip, i*64+ 5) >> 3;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*25+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 22 | (uint64_t)IPX(ip, i*64+7) << 47;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*25+ 3) = (uint64_t)IPW(ip, i*64+ 7) >> 17;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*64+ 9) << 33 | (uint64_t)IPX(ip, i*64+10) << 58;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*25+ 4) = (uint64_t)IPW(ip, i*64+10) >> 6;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*25+ 4) |= (uint64_t)IPV(ip, i*64+11) << 19 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*25+ 5) = (uint64_t)IPW(ip, i*64+12) >> 20;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*25+ 5) |= (uint64_t)IPV(ip, i*64+13) << 5;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*25+ 5) |= (uint64_t)IPV(ip, i*64+14) << 30 | (uint64_t)IPX(ip, i*64+15) << 55;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*25+ 6) = (uint64_t)IPW(ip, i*64+15) >> 9;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*25+ 6) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 41;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*25+ 7) = (uint64_t)IPW(ip, i*64+17) >> 23;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*64+18) << 2;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*64+19) << 27 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*25+ 8) = (uint64_t)IPW(ip, i*64+20) >> 12;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*25+ 8) |= (uint64_t)IPV(ip, i*64+21) << 13;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*25+ 8) |= (uint64_t)IPV(ip, i*64+22) << 38 | (uint64_t)IPX(ip, i*64+23) << 63;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*25+ 9) = (uint64_t)IPW(ip, i*64+23) >> 1;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*25+ 9) |= (uint64_t)IPV(ip, i*64+24) << 24 | (uint64_t)IPX(ip, i*64+25) << 49;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*25+10) = (uint64_t)IPW(ip, i*64+25) >> 15;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*64+26) << 10;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*64+27) << 35 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*25+11) = (uint64_t)IPW(ip, i*64+28) >> 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*25+11) |= (uint64_t)IPV(ip, i*64+29) << 21 | (uint64_t)IPX(ip, i*64+30) << 46;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*25+12) = (uint64_t)IPW(ip, i*64+30) >> 18;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*25+12) |= (uint64_t)IPV(ip, i*64+31) << 7;\
-}
-
-#define BITPACK64_25(ip, op, parm) { \
- BITBLK64_25(ip, 0, op, parm); IPI(ip); op += 25*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_26(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 26 | (uint64_t)IPX(ip, i*32+2) << 52;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 12;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 14 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*32+ 4) >> 24;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*32+ 5) << 2;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*32+ 6) << 28 | (uint64_t)IPX(ip, i*32+7) << 54;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*32+ 7) >> 10;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 42;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*32+ 9) >> 22;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*32+10) << 4;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*32+11) << 30 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*32+12) >> 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*32+13) << 18 | (uint64_t)IPX(ip, i*32+14) << 44;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*32+14) >> 20;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*32+15) << 6;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 58;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*13+ 7) = (uint64_t)IPW(ip, i*32+17) >> 6;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*13+ 7) |= (uint64_t)IPV(ip, i*32+18) << 20 | (uint64_t)IPX(ip, i*32+19) << 46;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*13+ 8) = (uint64_t)IPW(ip, i*32+19) >> 18;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*32+20) << 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*32+21) << 34 | (uint64_t)IPX(ip, i*32+22) << 60;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*13+ 9) = (uint64_t)IPW(ip, i*32+22) >> 4;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*13+ 9) |= (uint64_t)IPV(ip, i*32+23) << 22 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*13+10) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*13+10) |= (uint64_t)IPV(ip, i*32+25) << 10;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*13+10) |= (uint64_t)IPV(ip, i*32+26) << 36 | (uint64_t)IPX(ip, i*32+27) << 62;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*13+11) = (uint64_t)IPW(ip, i*32+27) >> 2;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*13+11) |= (uint64_t)IPV(ip, i*32+28) << 24 | (uint64_t)IPX(ip, i*32+29) << 50;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*13+12) = (uint64_t)IPW(ip, i*32+29) >> 14;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*32+30) << 12;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*32+31) << 38;\
-}
-
-#define BITPACK64_26(ip, op, parm) { \
- BITBLK64_26(ip, 0, op, parm); IPI(ip); op += 26*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_27(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*27+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*27+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 27 | (uint64_t)IPX(ip, i*64+2) << 54;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*27+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 10;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*27+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 17 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*27+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 20;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*27+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 7;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*27+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 34 | (uint64_t)IPX(ip, i*64+7) << 61;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*27+ 3) = (uint64_t)IPW(ip, i*64+ 7) >> 3;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*27+ 3) |= (uint64_t)IPV(ip, i*64+ 8) << 24 | (uint64_t)IPX(ip, i*64+9) << 51;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*27+ 4) = (uint64_t)IPW(ip, i*64+ 9) >> 13;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*27+ 4) |= (uint64_t)IPV(ip, i*64+10) << 14 | (uint64_t)IPX(ip, i*64+11) << 41;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*27+ 5) = (uint64_t)IPW(ip, i*64+11) >> 23;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*64+12) << 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*64+13) << 31 | (uint64_t)IPX(ip, i*64+14) << 58;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*27+ 6) = (uint64_t)IPW(ip, i*64+14) >> 6;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*27+ 6) |= (uint64_t)IPV(ip, i*64+15) << 21 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*27+ 7) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*27+ 7) |= (uint64_t)IPV(ip, i*64+17) << 11 | (uint64_t)IPX(ip, i*64+18) << 38;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*27+ 8) = (uint64_t)IPW(ip, i*64+18) >> 26;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*27+ 8) |= (uint64_t)IPV(ip, i*64+19) << 1;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*27+ 8) |= (uint64_t)IPV(ip, i*64+20) << 28 | (uint64_t)IPX(ip, i*64+21) << 55;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*27+ 9) = (uint64_t)IPW(ip, i*64+21) >> 9;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*27+ 9) |= (uint64_t)IPV(ip, i*64+22) << 18 | (uint64_t)IPX(ip, i*64+23) << 45;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*27+10) = (uint64_t)IPW(ip, i*64+23) >> 19;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*64+25) << 35 | (uint64_t)IPX(ip, i*64+26) << 62;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*27+11) = (uint64_t)IPW(ip, i*64+26) >> 2;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*27+11) |= (uint64_t)IPV(ip, i*64+27) << 25 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*27+12) = (uint64_t)IPW(ip, i*64+28) >> 12;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*27+12) |= (uint64_t)IPV(ip, i*64+29) << 15 | (uint64_t)IPX(ip, i*64+30) << 42;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*27+13) = (uint64_t)IPW(ip, i*64+30) >> 22;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*27+13) |= (uint64_t)IPV(ip, i*64+31) << 5;\
-}
-
-#define BITPACK64_27(ip, op, parm) { \
- BITBLK64_27(ip, 0, op, parm); IPI(ip); op += 27*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_28(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 28 | (uint64_t)IPX(ip, i*16+2) << 56;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*16+ 2) >> 8;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*16+ 3) << 20 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*16+ 4) >> 16;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*16+ 5) << 12 | (uint64_t)IPX(ip, i*16+6) << 40;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*16+ 6) >> 24;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*16+ 7) << 4;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*16+ 8) << 32 | (uint64_t)IPX(ip, i*16+9) << 60;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*16+ 9) >> 4;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*16+10) << 24 | (uint64_t)IPX(ip, i*16+11) << 52;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*16+11) >> 12;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*16+12) << 16 | (uint64_t)IPX(ip, i*16+13) << 44;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*16+13) >> 20;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*16+14) << 8;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*16+15) << 36;\
-}
-
-#define BITPACK64_28(ip, op, parm) { \
- BITBLK64_28(ip, 0, op, parm);\
- BITBLK64_28(ip, 1, op, parm); IPI(ip); op += 28*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_29(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*29+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*29+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 29 | (uint64_t)IPX(ip, i*64+2) << 58;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*29+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 6;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*29+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 23 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*29+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*29+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 17 | (uint64_t)IPX(ip, i*64+6) << 46;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*29+ 3) = (uint64_t)IPW(ip, i*64+ 6) >> 18;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*29+ 3) |= (uint64_t)IPV(ip, i*64+ 7) << 11 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*29+ 4) = (uint64_t)IPW(ip, i*64+ 8) >> 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*29+ 4) |= (uint64_t)IPV(ip, i*64+ 9) << 5;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*29+ 4) |= (uint64_t)IPV(ip, i*64+10) << 34 | (uint64_t)IPX(ip, i*64+11) << 63;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*29+ 5) = (uint64_t)IPW(ip, i*64+11) >> 1;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*29+ 5) |= (uint64_t)IPV(ip, i*64+12) << 28 | (uint64_t)IPX(ip, i*64+13) << 57;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*29+ 6) = (uint64_t)IPW(ip, i*64+13) >> 7;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*29+ 6) |= (uint64_t)IPV(ip, i*64+14) << 22 | (uint64_t)IPX(ip, i*64+15) << 51;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*29+ 7) = (uint64_t)IPW(ip, i*64+15) >> 13;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*29+ 7) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 45;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*29+ 8) = (uint64_t)IPW(ip, i*64+17) >> 19;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*29+ 8) |= (uint64_t)IPV(ip, i*64+18) << 10 | (uint64_t)IPX(ip, i*64+19) << 39;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*29+ 9) = (uint64_t)IPW(ip, i*64+19) >> 25;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*64+20) << 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*64+21) << 33 | (uint64_t)IPX(ip, i*64+22) << 62;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*29+10) = (uint64_t)IPW(ip, i*64+22) >> 2;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*29+10) |= (uint64_t)IPV(ip, i*64+23) << 27 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*29+11) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*29+11) |= (uint64_t)IPV(ip, i*64+25) << 21 | (uint64_t)IPX(ip, i*64+26) << 50;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*29+12) = (uint64_t)IPW(ip, i*64+26) >> 14;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*29+12) |= (uint64_t)IPV(ip, i*64+27) << 15 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*29+13) = (uint64_t)IPW(ip, i*64+28) >> 20;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*29+13) |= (uint64_t)IPV(ip, i*64+29) << 9 | (uint64_t)IPX(ip, i*64+30) << 38;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*29+14) = (uint64_t)IPW(ip, i*64+30) >> 26;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*29+14) |= (uint64_t)IPV(ip, i*64+31) << 3;\
-}
-
-#define BITPACK64_29(ip, op, parm) { \
- BITBLK64_29(ip, 0, op, parm); IPI(ip); op += 29*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_30(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 30 | (uint64_t)IPX(ip, i*32+2) << 60;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 4;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 26 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*32+ 4) >> 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*32+ 5) << 22 | (uint64_t)IPX(ip, i*32+6) << 52;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*32+ 6) >> 12;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*32+ 7) << 18 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*32+ 9) << 14 | (uint64_t)IPX(ip, i*32+10) << 44;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*32+10) >> 20;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*32+11) << 10 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*32+12) >> 24;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*32+13) << 6 | (uint64_t)IPX(ip, i*32+14) << 36;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*32+14) >> 28;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*32+15) << 2;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 62;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*15+ 8) = (uint64_t)IPW(ip, i*32+17) >> 2;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*15+ 8) |= (uint64_t)IPV(ip, i*32+18) << 28 | (uint64_t)IPX(ip, i*32+19) << 58;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*15+ 9) = (uint64_t)IPW(ip, i*32+19) >> 6;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*15+ 9) |= (uint64_t)IPV(ip, i*32+20) << 24 | (uint64_t)IPX(ip, i*32+21) << 54;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*15+10) = (uint64_t)IPW(ip, i*32+21) >> 10;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*15+10) |= (uint64_t)IPV(ip, i*32+22) << 20 | (uint64_t)IPX(ip, i*32+23) << 50;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*15+11) = (uint64_t)IPW(ip, i*32+23) >> 14;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*15+11) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 46;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*15+12) = (uint64_t)IPW(ip, i*32+25) >> 18;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*15+12) |= (uint64_t)IPV(ip, i*32+26) << 12 | (uint64_t)IPX(ip, i*32+27) << 42;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*15+13) = (uint64_t)IPW(ip, i*32+27) >> 22;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*15+13) |= (uint64_t)IPV(ip, i*32+28) << 8 | (uint64_t)IPX(ip, i*32+29) << 38;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*15+14) = (uint64_t)IPW(ip, i*32+29) >> 26;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*32+30) << 4;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*32+31) << 34;\
-}
-
-#define BITPACK64_30(ip, op, parm) { \
- BITBLK64_30(ip, 0, op, parm); IPI(ip); op += 30*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_31(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*31+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*31+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 31 | (uint64_t)IPX(ip, i*64+2) << 62;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*31+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 2;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*31+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 29 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*31+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*31+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 27 | (uint64_t)IPX(ip, i*64+6) << 58;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*31+ 3) = (uint64_t)IPW(ip, i*64+ 6) >> 6;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*31+ 3) |= (uint64_t)IPV(ip, i*64+ 7) << 25 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*31+ 4) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*31+ 4) |= (uint64_t)IPV(ip, i*64+ 9) << 23 | (uint64_t)IPX(ip, i*64+10) << 54;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*31+ 5) = (uint64_t)IPW(ip, i*64+10) >> 10;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*31+ 5) |= (uint64_t)IPV(ip, i*64+11) << 21 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*31+ 6) = (uint64_t)IPW(ip, i*64+12) >> 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*31+ 6) |= (uint64_t)IPV(ip, i*64+13) << 19 | (uint64_t)IPX(ip, i*64+14) << 50;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*31+ 7) = (uint64_t)IPW(ip, i*64+14) >> 14;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*31+ 7) |= (uint64_t)IPV(ip, i*64+15) << 17 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*31+ 8) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*31+ 8) |= (uint64_t)IPV(ip, i*64+17) << 15 | (uint64_t)IPX(ip, i*64+18) << 46;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*31+ 9) = (uint64_t)IPW(ip, i*64+18) >> 18;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*31+ 9) |= (uint64_t)IPV(ip, i*64+19) << 13 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*31+10) = (uint64_t)IPW(ip, i*64+20) >> 20;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*31+10) |= (uint64_t)IPV(ip, i*64+21) << 11 | (uint64_t)IPX(ip, i*64+22) << 42;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*31+11) = (uint64_t)IPW(ip, i*64+22) >> 22;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*31+11) |= (uint64_t)IPV(ip, i*64+23) << 9 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*31+12) = (uint64_t)IPW(ip, i*64+24) >> 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*31+12) |= (uint64_t)IPV(ip, i*64+25) << 7 | (uint64_t)IPX(ip, i*64+26) << 38;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*31+13) = (uint64_t)IPW(ip, i*64+26) >> 26;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*31+13) |= (uint64_t)IPV(ip, i*64+27) << 5 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*31+14) = (uint64_t)IPW(ip, i*64+28) >> 28;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*31+14) |= (uint64_t)IPV(ip, i*64+29) << 3 | (uint64_t)IPX(ip, i*64+30) << 34;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*31+15) = (uint64_t)IPW(ip, i*64+30) >> 30;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*31+15) |= (uint64_t)IPV(ip, i*64+31) << 1;\
-}
-
-#define BITPACK64_31(ip, op, parm) { \
- BITBLK64_31(ip, 0, op, parm); IPI(ip); op += 31*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_32(ip, i, op, parm) { \
- IP9(ip, i*2+ 0, parm); *(uint32_t *)(op+i*8+ 0) = IPV(ip, i*2+ 0);\
- IP9(ip, i*2+ 1, parm); *(uint32_t *)(op+i*8+ 4) = IPV(ip, i*2+ 1);;\
-}
-
-#define BITPACK64_32(ip, op, parm) { \
- BITBLK64_32(ip, 0, op, parm);\
- BITBLK64_32(ip, 1, op, parm);\
- BITBLK64_32(ip, 2, op, parm);\
- BITBLK64_32(ip, 3, op, parm);\
- BITBLK64_32(ip, 4, op, parm);\
- BITBLK64_32(ip, 5, op, parm);\
- BITBLK64_32(ip, 6, op, parm);\
- BITBLK64_32(ip, 7, op, parm);\
- BITBLK64_32(ip, 8, op, parm);\
- BITBLK64_32(ip, 9, op, parm);\
- BITBLK64_32(ip, 10, op, parm);\
- BITBLK64_32(ip, 11, op, parm);\
- BITBLK64_32(ip, 12, op, parm);\
- BITBLK64_32(ip, 13, op, parm);\
- BITBLK64_32(ip, 14, op, parm);\
- BITBLK64_32(ip, 15, op, parm); IPI(ip); op += 32*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_33(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*33+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 33;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*33+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 31;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*33+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 2 | (uint64_t)IPX(ip, i*64+3) << 35;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*33+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 29;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*33+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 4 | (uint64_t)IPX(ip, i*64+5) << 37;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*33+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 27;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*33+ 3) |= (uint64_t)IPV(ip, i*64+ 6) << 6 | (uint64_t)IPX(ip, i*64+7) << 39;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*33+ 4) = (uint64_t)IPW(ip, i*64+ 7) >> 25;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*33+ 4) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 41;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*33+ 5) = (uint64_t)IPW(ip, i*64+ 9) >> 23;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*33+ 5) |= (uint64_t)IPV(ip, i*64+10) << 10 | (uint64_t)IPX(ip, i*64+11) << 43;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*33+ 6) = (uint64_t)IPW(ip, i*64+11) >> 21;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*33+ 6) |= (uint64_t)IPV(ip, i*64+12) << 12 | (uint64_t)IPX(ip, i*64+13) << 45;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*33+ 7) = (uint64_t)IPW(ip, i*64+13) >> 19;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*33+ 7) |= (uint64_t)IPV(ip, i*64+14) << 14 | (uint64_t)IPX(ip, i*64+15) << 47;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*33+ 8) = (uint64_t)IPW(ip, i*64+15) >> 17;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*33+ 8) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 49;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*33+ 9) = (uint64_t)IPW(ip, i*64+17) >> 15;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*33+ 9) |= (uint64_t)IPV(ip, i*64+18) << 18 | (uint64_t)IPX(ip, i*64+19) << 51;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*33+10) = (uint64_t)IPW(ip, i*64+19) >> 13;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*33+10) |= (uint64_t)IPV(ip, i*64+20) << 20 | (uint64_t)IPX(ip, i*64+21) << 53;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*33+11) = (uint64_t)IPW(ip, i*64+21) >> 11;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*33+11) |= (uint64_t)IPV(ip, i*64+22) << 22 | (uint64_t)IPX(ip, i*64+23) << 55;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*33+12) = (uint64_t)IPW(ip, i*64+23) >> 9;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*33+12) |= (uint64_t)IPV(ip, i*64+24) << 24 | (uint64_t)IPX(ip, i*64+25) << 57;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*33+13) = (uint64_t)IPW(ip, i*64+25) >> 7;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*33+13) |= (uint64_t)IPV(ip, i*64+26) << 26 | (uint64_t)IPX(ip, i*64+27) << 59;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*33+14) = (uint64_t)IPW(ip, i*64+27) >> 5;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*33+14) |= (uint64_t)IPV(ip, i*64+28) << 28 | (uint64_t)IPX(ip, i*64+29) << 61;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*33+15) = (uint64_t)IPW(ip, i*64+29) >> 3;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*33+15) |= (uint64_t)IPV(ip, i*64+30) << 30 | (uint64_t)IPX(ip, i*64+31) << 63;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*33+16) = (uint64_t)IPW(ip, i*64+31) >> 1;\
-}
-
-#define BITPACK64_33(ip, op, parm) { \
- BITBLK64_33(ip, 0, op, parm); IPI(ip); op += 33*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_34(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*17+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 34;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*17+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 30;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 4 | (uint64_t)IPX(ip, i*32+3) << 38;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*17+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 26;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*32+ 4) << 8 | (uint64_t)IPX(ip, i*32+5) << 42;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*17+ 3) = (uint64_t)IPW(ip, i*32+ 5) >> 22;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*32+ 6) << 12 | (uint64_t)IPX(ip, i*32+7) << 46;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*17+ 4) = (uint64_t)IPW(ip, i*32+ 7) >> 18;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 50;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*17+ 5) = (uint64_t)IPW(ip, i*32+ 9) >> 14;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*32+10) << 20 | (uint64_t)IPX(ip, i*32+11) << 54;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*17+ 6) = (uint64_t)IPW(ip, i*32+11) >> 10;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*32+12) << 24 | (uint64_t)IPX(ip, i*32+13) << 58;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*17+ 7) = (uint64_t)IPW(ip, i*32+13) >> 6;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*32+14) << 28 | (uint64_t)IPX(ip, i*32+15) << 62;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*17+ 8) = (uint64_t)IPW(ip, i*32+15) >> 2 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*17+ 9) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*17+ 9) |= (uint64_t)IPV(ip, i*32+17) << 2 | (uint64_t)IPX(ip, i*32+18) << 36;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*17+10) = (uint64_t)IPW(ip, i*32+18) >> 28;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*17+10) |= (uint64_t)IPV(ip, i*32+19) << 6 | (uint64_t)IPX(ip, i*32+20) << 40;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*17+11) = (uint64_t)IPW(ip, i*32+20) >> 24;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*17+11) |= (uint64_t)IPV(ip, i*32+21) << 10 | (uint64_t)IPX(ip, i*32+22) << 44;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*17+12) = (uint64_t)IPW(ip, i*32+22) >> 20;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*17+12) |= (uint64_t)IPV(ip, i*32+23) << 14 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*17+13) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*17+13) |= (uint64_t)IPV(ip, i*32+25) << 18 | (uint64_t)IPX(ip, i*32+26) << 52;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*17+14) = (uint64_t)IPW(ip, i*32+26) >> 12;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*17+14) |= (uint64_t)IPV(ip, i*32+27) << 22 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*17+15) = (uint64_t)IPW(ip, i*32+28) >> 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*17+15) |= (uint64_t)IPV(ip, i*32+29) << 26 | (uint64_t)IPX(ip, i*32+30) << 60;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*17+16) = (uint64_t)IPW(ip, i*32+30) >> 4;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*17+16) |= (uint64_t)IPV(ip, i*32+31) << 30;\
-}
-
-#define BITPACK64_34(ip, op, parm) { \
- BITBLK64_34(ip, 0, op, parm); IPI(ip); op += 34*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_35(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*35+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 35;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*35+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 29;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*35+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 6 | (uint64_t)IPX(ip, i*64+3) << 41;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*35+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 23;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*35+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 12 | (uint64_t)IPX(ip, i*64+5) << 47;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*35+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 17;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*35+ 3) |= (uint64_t)IPV(ip, i*64+ 6) << 18 | (uint64_t)IPX(ip, i*64+7) << 53;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*35+ 4) = (uint64_t)IPW(ip, i*64+ 7) >> 11;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*35+ 4) |= (uint64_t)IPV(ip, i*64+ 8) << 24 | (uint64_t)IPX(ip, i*64+9) << 59;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*35+ 5) = (uint64_t)IPW(ip, i*64+ 9) >> 5 | (uint64_t)IPX(ip, i*64+10) << 30;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*35+ 6) = (uint64_t)IPW(ip, i*64+10) >> 34;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*35+ 6) |= (uint64_t)IPV(ip, i*64+11) << 1 | (uint64_t)IPX(ip, i*64+12) << 36;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*35+ 7) = (uint64_t)IPW(ip, i*64+12) >> 28;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*35+ 7) |= (uint64_t)IPV(ip, i*64+13) << 7 | (uint64_t)IPX(ip, i*64+14) << 42;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*35+ 8) = (uint64_t)IPW(ip, i*64+14) >> 22;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*35+ 8) |= (uint64_t)IPV(ip, i*64+15) << 13 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*35+ 9) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*35+ 9) |= (uint64_t)IPV(ip, i*64+17) << 19 | (uint64_t)IPX(ip, i*64+18) << 54;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*35+10) = (uint64_t)IPW(ip, i*64+18) >> 10;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*35+10) |= (uint64_t)IPV(ip, i*64+19) << 25 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*35+11) = (uint64_t)IPW(ip, i*64+20) >> 4 | (uint64_t)IPX(ip, i*64+21) << 31;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*35+12) = (uint64_t)IPW(ip, i*64+21) >> 33;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*35+12) |= (uint64_t)IPV(ip, i*64+22) << 2 | (uint64_t)IPX(ip, i*64+23) << 37;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*35+13) = (uint64_t)IPW(ip, i*64+23) >> 27;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*35+13) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 43;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*35+14) = (uint64_t)IPW(ip, i*64+25) >> 21;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*35+14) |= (uint64_t)IPV(ip, i*64+26) << 14 | (uint64_t)IPX(ip, i*64+27) << 49;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*35+15) = (uint64_t)IPW(ip, i*64+27) >> 15;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*35+15) |= (uint64_t)IPV(ip, i*64+28) << 20 | (uint64_t)IPX(ip, i*64+29) << 55;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*35+16) = (uint64_t)IPW(ip, i*64+29) >> 9;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*35+16) |= (uint64_t)IPV(ip, i*64+30) << 26 | (uint64_t)IPX(ip, i*64+31) << 61;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*35+17) = (uint64_t)IPW(ip, i*64+31) >> 3;\
-}
-
-#define BITPACK64_35(ip, op, parm) { \
- BITBLK64_35(ip, 0, op, parm); IPI(ip); op += 35*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_36(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 36;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 28;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*16+ 2) << 8 | (uint64_t)IPX(ip, i*16+3) << 44;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*16+ 3) >> 20;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*16+ 4) << 16 | (uint64_t)IPX(ip, i*16+5) << 52;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*16+ 5) >> 12;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*16+ 6) << 24 | (uint64_t)IPX(ip, i*16+7) << 60;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*16+ 7) >> 4 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*9+ 5) = (uint64_t)IPW(ip, i*16+ 8) >> 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*16+ 9) << 4 | (uint64_t)IPX(ip, i*16+10) << 40;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*9+ 6) = (uint64_t)IPW(ip, i*16+10) >> 24;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*16+11) << 12 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*9+ 7) = (uint64_t)IPW(ip, i*16+12) >> 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*16+13) << 20 | (uint64_t)IPX(ip, i*16+14) << 56;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*9+ 8) = (uint64_t)IPW(ip, i*16+14) >> 8;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*16+15) << 28;\
-}
-
-#define BITPACK64_36(ip, op, parm) { \
- BITBLK64_36(ip, 0, op, parm);\
- BITBLK64_36(ip, 1, op, parm); IPI(ip); op += 36*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_37(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*37+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 37;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*37+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 27;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*37+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 10 | (uint64_t)IPX(ip, i*64+3) << 47;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*37+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 17;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*37+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 20 | (uint64_t)IPX(ip, i*64+5) << 57;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*37+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 7 | (uint64_t)IPX(ip, i*64+6) << 30;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*37+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 34;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*37+ 4) |= (uint64_t)IPV(ip, i*64+ 7) << 3 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*37+ 5) = (uint64_t)IPW(ip, i*64+ 8) >> 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*37+ 5) |= (uint64_t)IPV(ip, i*64+ 9) << 13 | (uint64_t)IPX(ip, i*64+10) << 50;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*37+ 6) = (uint64_t)IPW(ip, i*64+10) >> 14;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*37+ 6) |= (uint64_t)IPV(ip, i*64+11) << 23 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*37+ 7) = (uint64_t)IPW(ip, i*64+12) >> 4 | (uint64_t)IPX(ip, i*64+13) << 33;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*37+ 8) = (uint64_t)IPW(ip, i*64+13) >> 31;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*37+ 8) |= (uint64_t)IPV(ip, i*64+14) << 6 | (uint64_t)IPX(ip, i*64+15) << 43;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*37+ 9) = (uint64_t)IPW(ip, i*64+15) >> 21;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*37+ 9) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 53;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*37+10) = (uint64_t)IPW(ip, i*64+17) >> 11;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*37+10) |= (uint64_t)IPV(ip, i*64+18) << 26 | (uint64_t)IPX(ip, i*64+19) << 63;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*37+11) = (uint64_t)IPW(ip, i*64+19) >> 1 | (uint64_t)IPX(ip, i*64+20) << 36;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*37+12) = (uint64_t)IPW(ip, i*64+20) >> 28;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*37+12) |= (uint64_t)IPV(ip, i*64+21) << 9 | (uint64_t)IPX(ip, i*64+22) << 46;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*37+13) = (uint64_t)IPW(ip, i*64+22) >> 18;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*37+13) |= (uint64_t)IPV(ip, i*64+23) << 19 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*37+14) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 29;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*37+15) = (uint64_t)IPW(ip, i*64+25) >> 35;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*37+15) |= (uint64_t)IPV(ip, i*64+26) << 2 | (uint64_t)IPX(ip, i*64+27) << 39;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*37+16) = (uint64_t)IPW(ip, i*64+27) >> 25;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*37+16) |= (uint64_t)IPV(ip, i*64+28) << 12 | (uint64_t)IPX(ip, i*64+29) << 49;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*37+17) = (uint64_t)IPW(ip, i*64+29) >> 15;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*37+17) |= (uint64_t)IPV(ip, i*64+30) << 22 | (uint64_t)IPX(ip, i*64+31) << 59;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*37+18) = (uint64_t)IPW(ip, i*64+31) >> 5;\
-}
-
-#define BITPACK64_37(ip, op, parm) { \
- BITBLK64_37(ip, 0, op, parm); IPI(ip); op += 37*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_38(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*19+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 38;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*19+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 26;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 12 | (uint64_t)IPX(ip, i*32+3) << 50;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*19+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 14;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*32+ 4) << 24 | (uint64_t)IPX(ip, i*32+5) << 62;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*19+ 3) = (uint64_t)IPW(ip, i*32+ 5) >> 2 | (uint64_t)IPX(ip, i*32+6) << 36;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*19+ 4) = (uint64_t)IPW(ip, i*32+ 6) >> 28;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*32+ 7) << 10 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*19+ 5) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*32+ 9) << 22 | (uint64_t)IPX(ip, i*32+10) << 60;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*19+ 6) = (uint64_t)IPW(ip, i*32+10) >> 4 | (uint64_t)IPX(ip, i*32+11) << 34;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*19+ 7) = (uint64_t)IPW(ip, i*32+11) >> 30;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*32+12) << 8 | (uint64_t)IPX(ip, i*32+13) << 46;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*19+ 8) = (uint64_t)IPW(ip, i*32+13) >> 18;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*32+14) << 20 | (uint64_t)IPX(ip, i*32+15) << 58;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*19+ 9) = (uint64_t)IPW(ip, i*32+15) >> 6 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*19+10) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*19+10) |= (uint64_t)IPV(ip, i*32+17) << 6 | (uint64_t)IPX(ip, i*32+18) << 44;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*19+11) = (uint64_t)IPW(ip, i*32+18) >> 20;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*19+11) |= (uint64_t)IPV(ip, i*32+19) << 18 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*19+12) = (uint64_t)IPW(ip, i*32+20) >> 8 | (uint64_t)IPX(ip, i*32+21) << 30;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*19+13) = (uint64_t)IPW(ip, i*32+21) >> 34;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*19+13) |= (uint64_t)IPV(ip, i*32+22) << 4 | (uint64_t)IPX(ip, i*32+23) << 42;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*19+14) = (uint64_t)IPW(ip, i*32+23) >> 22;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*19+14) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 54;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*19+15) = (uint64_t)IPW(ip, i*32+25) >> 10 | (uint64_t)IPX(ip, i*32+26) << 28;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*19+16) = (uint64_t)IPW(ip, i*32+26) >> 36;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*19+16) |= (uint64_t)IPV(ip, i*32+27) << 2 | (uint64_t)IPX(ip, i*32+28) << 40;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*19+17) = (uint64_t)IPW(ip, i*32+28) >> 24;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*19+17) |= (uint64_t)IPV(ip, i*32+29) << 14 | (uint64_t)IPX(ip, i*32+30) << 52;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*19+18) = (uint64_t)IPW(ip, i*32+30) >> 12;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*19+18) |= (uint64_t)IPV(ip, i*32+31) << 26;\
-}
-
-#define BITPACK64_38(ip, op, parm) { \
- BITBLK64_38(ip, 0, op, parm); IPI(ip); op += 38*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_39(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*39+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 39;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*39+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 25;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*39+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 14 | (uint64_t)IPX(ip, i*64+3) << 53;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*39+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 11 | (uint64_t)IPX(ip, i*64+4) << 28;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*39+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 36;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*39+ 3) |= (uint64_t)IPV(ip, i*64+ 5) << 3 | (uint64_t)IPX(ip, i*64+6) << 42;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*39+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 22;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*39+ 4) |= (uint64_t)IPV(ip, i*64+ 7) << 17 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*39+ 5) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 31;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*39+ 6) = (uint64_t)IPW(ip, i*64+ 9) >> 33;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*39+ 6) |= (uint64_t)IPV(ip, i*64+10) << 6 | (uint64_t)IPX(ip, i*64+11) << 45;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*39+ 7) = (uint64_t)IPW(ip, i*64+11) >> 19;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*39+ 7) |= (uint64_t)IPV(ip, i*64+12) << 20 | (uint64_t)IPX(ip, i*64+13) << 59;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*39+ 8) = (uint64_t)IPW(ip, i*64+13) >> 5 | (uint64_t)IPX(ip, i*64+14) << 34;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*39+ 9) = (uint64_t)IPW(ip, i*64+14) >> 30;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*39+ 9) |= (uint64_t)IPV(ip, i*64+15) << 9 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*39+10) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*39+10) |= (uint64_t)IPV(ip, i*64+17) << 23 | (uint64_t)IPX(ip, i*64+18) << 62;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*39+11) = (uint64_t)IPW(ip, i*64+18) >> 2 | (uint64_t)IPX(ip, i*64+19) << 37;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*39+12) = (uint64_t)IPW(ip, i*64+19) >> 27;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*39+12) |= (uint64_t)IPV(ip, i*64+20) << 12 | (uint64_t)IPX(ip, i*64+21) << 51;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*39+13) = (uint64_t)IPW(ip, i*64+21) >> 13 | (uint64_t)IPX(ip, i*64+22) << 26;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*39+14) = (uint64_t)IPW(ip, i*64+22) >> 38;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*39+14) |= (uint64_t)IPV(ip, i*64+23) << 1 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*39+15) = (uint64_t)IPW(ip, i*64+24) >> 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*39+15) |= (uint64_t)IPV(ip, i*64+25) << 15 | (uint64_t)IPX(ip, i*64+26) << 54;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*39+16) = (uint64_t)IPW(ip, i*64+26) >> 10 | (uint64_t)IPX(ip, i*64+27) << 29;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*39+17) = (uint64_t)IPW(ip, i*64+27) >> 35;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*39+17) |= (uint64_t)IPV(ip, i*64+28) << 4 | (uint64_t)IPX(ip, i*64+29) << 43;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*39+18) = (uint64_t)IPW(ip, i*64+29) >> 21;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*39+18) |= (uint64_t)IPV(ip, i*64+30) << 18 | (uint64_t)IPX(ip, i*64+31) << 57;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*39+19) = (uint64_t)IPW(ip, i*64+31) >> 7;\
-}
-
-#define BITPACK64_39(ip, op, parm) { \
- BITBLK64_39(ip, 0, op, parm); IPI(ip); op += 39*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_40(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*8+ 0) | (uint64_t)IPX(ip, i*8+1) << 40;\
- IP64(ip, i*8+ 1, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*8+ 1) >> 24;\
- IP9(ip, i*8+ 2, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*8+ 2) << 16 | (uint64_t)IPX(ip, i*8+3) << 56;\
- IP64(ip, i*8+ 3, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*8+ 3) >> 8 | (uint64_t)IPX(ip, i*8+4) << 32;\
- IP64(ip, i*8+ 4, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*8+ 4) >> 32;\
- IP9(ip, i*8+ 5, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*8+ 5) << 8 | (uint64_t)IPX(ip, i*8+6) << 48;\
- IP64(ip, i*8+ 6, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*8+ 6) >> 16;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*8+ 7) << 24;\
-}
-
-#define BITPACK64_40(ip, op, parm) { \
- BITBLK64_40(ip, 0, op, parm);\
- BITBLK64_40(ip, 1, op, parm);\
- BITBLK64_40(ip, 2, op, parm);\
- BITBLK64_40(ip, 3, op, parm); IPI(ip); op += 40*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_41(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*41+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 41;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*41+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 23;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*41+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 18 | (uint64_t)IPX(ip, i*64+3) << 59;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*41+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 5 | (uint64_t)IPX(ip, i*64+4) << 36;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*41+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 28;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*41+ 3) |= (uint64_t)IPV(ip, i*64+ 5) << 13 | (uint64_t)IPX(ip, i*64+6) << 54;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*41+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 10 | (uint64_t)IPX(ip, i*64+7) << 31;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*41+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 33;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*41+ 5) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 49;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*41+ 6) = (uint64_t)IPW(ip, i*64+ 9) >> 15 | (uint64_t)IPX(ip, i*64+10) << 26;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*41+ 7) = (uint64_t)IPW(ip, i*64+10) >> 38;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*41+ 7) |= (uint64_t)IPV(ip, i*64+11) << 3 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*41+ 8) = (uint64_t)IPW(ip, i*64+12) >> 20;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*41+ 8) |= (uint64_t)IPV(ip, i*64+13) << 21 | (uint64_t)IPX(ip, i*64+14) << 62;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*41+ 9) = (uint64_t)IPW(ip, i*64+14) >> 2 | (uint64_t)IPX(ip, i*64+15) << 39;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*41+10) = (uint64_t)IPW(ip, i*64+15) >> 25;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*41+10) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 57;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*41+11) = (uint64_t)IPW(ip, i*64+17) >> 7 | (uint64_t)IPX(ip, i*64+18) << 34;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*41+12) = (uint64_t)IPW(ip, i*64+18) >> 30;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*41+12) |= (uint64_t)IPV(ip, i*64+19) << 11 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*41+13) = (uint64_t)IPW(ip, i*64+20) >> 12 | (uint64_t)IPX(ip, i*64+21) << 29;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*41+14) = (uint64_t)IPW(ip, i*64+21) >> 35;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*41+14) |= (uint64_t)IPV(ip, i*64+22) << 6 | (uint64_t)IPX(ip, i*64+23) << 47;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*41+15) = (uint64_t)IPW(ip, i*64+23) >> 17 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*41+16) = (uint64_t)IPW(ip, i*64+24) >> 40;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*41+16) |= (uint64_t)IPV(ip, i*64+25) << 1 | (uint64_t)IPX(ip, i*64+26) << 42;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*41+17) = (uint64_t)IPW(ip, i*64+26) >> 22;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*41+17) |= (uint64_t)IPV(ip, i*64+27) << 19 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*41+18) = (uint64_t)IPW(ip, i*64+28) >> 4 | (uint64_t)IPX(ip, i*64+29) << 37;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*41+19) = (uint64_t)IPW(ip, i*64+29) >> 27;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*41+19) |= (uint64_t)IPV(ip, i*64+30) << 14 | (uint64_t)IPX(ip, i*64+31) << 55;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*41+20) = (uint64_t)IPW(ip, i*64+31) >> 9;\
-}
-
-#define BITPACK64_41(ip, op, parm) { \
- BITBLK64_41(ip, 0, op, parm); IPI(ip); op += 41*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_42(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*21+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 42;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*21+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 22;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 20 | (uint64_t)IPX(ip, i*32+3) << 62;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*21+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 2 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*21+ 3) = (uint64_t)IPW(ip, i*32+ 4) >> 24;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*32+ 5) << 18 | (uint64_t)IPX(ip, i*32+6) << 60;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*21+ 4) = (uint64_t)IPW(ip, i*32+ 6) >> 4 | (uint64_t)IPX(ip, i*32+7) << 38;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*21+ 5) = (uint64_t)IPW(ip, i*32+ 7) >> 26;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 58;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*21+ 6) = (uint64_t)IPW(ip, i*32+ 9) >> 6 | (uint64_t)IPX(ip, i*32+10) << 36;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*21+ 7) = (uint64_t)IPW(ip, i*32+10) >> 28;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*32+11) << 14 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*21+ 8) = (uint64_t)IPW(ip, i*32+12) >> 8 | (uint64_t)IPX(ip, i*32+13) << 34;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*21+ 9) = (uint64_t)IPW(ip, i*32+13) >> 30;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*32+14) << 12 | (uint64_t)IPX(ip, i*32+15) << 54;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*21+10) = (uint64_t)IPW(ip, i*32+15) >> 10 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*21+11) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*21+11) |= (uint64_t)IPV(ip, i*32+17) << 10 | (uint64_t)IPX(ip, i*32+18) << 52;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*21+12) = (uint64_t)IPW(ip, i*32+18) >> 12 | (uint64_t)IPX(ip, i*32+19) << 30;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*21+13) = (uint64_t)IPW(ip, i*32+19) >> 34;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*21+13) |= (uint64_t)IPV(ip, i*32+20) << 8 | (uint64_t)IPX(ip, i*32+21) << 50;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*21+14) = (uint64_t)IPW(ip, i*32+21) >> 14 | (uint64_t)IPX(ip, i*32+22) << 28;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*21+15) = (uint64_t)IPW(ip, i*32+22) >> 36;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*21+15) |= (uint64_t)IPV(ip, i*32+23) << 6 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*21+16) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 26;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*21+17) = (uint64_t)IPW(ip, i*32+25) >> 38;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*21+17) |= (uint64_t)IPV(ip, i*32+26) << 4 | (uint64_t)IPX(ip, i*32+27) << 46;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*21+18) = (uint64_t)IPW(ip, i*32+27) >> 18 | (uint64_t)IPX(ip, i*32+28) << 24;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*21+19) = (uint64_t)IPW(ip, i*32+28) >> 40;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*21+19) |= (uint64_t)IPV(ip, i*32+29) << 2 | (uint64_t)IPX(ip, i*32+30) << 44;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*21+20) = (uint64_t)IPW(ip, i*32+30) >> 20;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*21+20) |= (uint64_t)IPV(ip, i*32+31) << 22;\
-}
-
-#define BITPACK64_42(ip, op, parm) { \
- BITBLK64_42(ip, 0, op, parm); IPI(ip); op += 42*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_43(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*43+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 43;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*43+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 21 | (uint64_t)IPX(ip, i*64+2) << 22;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*43+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 42;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*43+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 1 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*43+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 20 | (uint64_t)IPX(ip, i*64+5) << 23;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*43+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 41;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*43+ 4) |= (uint64_t)IPV(ip, i*64+ 6) << 2 | (uint64_t)IPX(ip, i*64+7) << 45;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*43+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 19 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*43+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 40;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*43+ 6) |= (uint64_t)IPV(ip, i*64+ 9) << 3 | (uint64_t)IPX(ip, i*64+10) << 46;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*43+ 7) = (uint64_t)IPW(ip, i*64+10) >> 18 | (uint64_t)IPX(ip, i*64+11) << 25;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*43+ 8) = (uint64_t)IPW(ip, i*64+11) >> 39;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*43+ 8) |= (uint64_t)IPV(ip, i*64+12) << 4 | (uint64_t)IPX(ip, i*64+13) << 47;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*43+ 9) = (uint64_t)IPW(ip, i*64+13) >> 17 | (uint64_t)IPX(ip, i*64+14) << 26;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*43+10) = (uint64_t)IPW(ip, i*64+14) >> 38;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*43+10) |= (uint64_t)IPV(ip, i*64+15) << 5 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*43+11) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 27;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*43+12) = (uint64_t)IPW(ip, i*64+17) >> 37;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*43+12) |= (uint64_t)IPV(ip, i*64+18) << 6 | (uint64_t)IPX(ip, i*64+19) << 49;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*43+13) = (uint64_t)IPW(ip, i*64+19) >> 15 | (uint64_t)IPX(ip, i*64+20) << 28;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*43+14) = (uint64_t)IPW(ip, i*64+20) >> 36;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*43+14) |= (uint64_t)IPV(ip, i*64+21) << 7 | (uint64_t)IPX(ip, i*64+22) << 50;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*43+15) = (uint64_t)IPW(ip, i*64+22) >> 14 | (uint64_t)IPX(ip, i*64+23) << 29;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*43+16) = (uint64_t)IPW(ip, i*64+23) >> 35;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*43+16) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 51;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*43+17) = (uint64_t)IPW(ip, i*64+25) >> 13 | (uint64_t)IPX(ip, i*64+26) << 30;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*43+18) = (uint64_t)IPW(ip, i*64+26) >> 34;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*43+18) |= (uint64_t)IPV(ip, i*64+27) << 9 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*43+19) = (uint64_t)IPW(ip, i*64+28) >> 12 | (uint64_t)IPX(ip, i*64+29) << 31;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*43+20) = (uint64_t)IPW(ip, i*64+29) >> 33;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*43+20) |= (uint64_t)IPV(ip, i*64+30) << 10 | (uint64_t)IPX(ip, i*64+31) << 53;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*43+21) = (uint64_t)IPW(ip, i*64+31) >> 11;\
-}
-
-#define BITPACK64_43(ip, op, parm) { \
- BITBLK64_43(ip, 0, op, parm); IPI(ip); op += 43*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_44(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 44;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 20 | (uint64_t)IPX(ip, i*16+2) << 24;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 40;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*16+ 3) << 4 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*16+ 4) >> 16 | (uint64_t)IPX(ip, i*16+5) << 28;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*16+ 5) >> 36;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*16+ 6) << 8 | (uint64_t)IPX(ip, i*16+7) << 52;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*16+ 7) >> 12 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*11+ 6) = (uint64_t)IPW(ip, i*16+ 8) >> 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*16+ 9) << 12 | (uint64_t)IPX(ip, i*16+10) << 56;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*11+ 7) = (uint64_t)IPW(ip, i*16+10) >> 8 | (uint64_t)IPX(ip, i*16+11) << 36;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*11+ 8) = (uint64_t)IPW(ip, i*16+11) >> 28;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*16+12) << 16 | (uint64_t)IPX(ip, i*16+13) << 60;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*11+ 9) = (uint64_t)IPW(ip, i*16+13) >> 4 | (uint64_t)IPX(ip, i*16+14) << 40;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*11+10) = (uint64_t)IPW(ip, i*16+14) >> 24;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*16+15) << 20;\
-}
-
-#define BITPACK64_44(ip, op, parm) { \
- BITBLK64_44(ip, 0, op, parm);\
- BITBLK64_44(ip, 1, op, parm); IPI(ip); op += 44*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_45(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*45+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 45;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*45+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 19 | (uint64_t)IPX(ip, i*64+2) << 26;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*45+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 38;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*45+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 7 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*45+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 12 | (uint64_t)IPX(ip, i*64+5) << 33;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*45+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 31;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*45+ 4) |= (uint64_t)IPV(ip, i*64+ 6) << 14 | (uint64_t)IPX(ip, i*64+7) << 59;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*45+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 5 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*45+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 21;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*45+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 43;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*45+ 7) |= (uint64_t)IPV(ip, i*64+10) << 2 | (uint64_t)IPX(ip, i*64+11) << 47;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*45+ 8) = (uint64_t)IPW(ip, i*64+11) >> 17 | (uint64_t)IPX(ip, i*64+12) << 28;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*45+ 9) = (uint64_t)IPW(ip, i*64+12) >> 36;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*45+ 9) |= (uint64_t)IPV(ip, i*64+13) << 9 | (uint64_t)IPX(ip, i*64+14) << 54;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*45+10) = (uint64_t)IPW(ip, i*64+14) >> 10 | (uint64_t)IPX(ip, i*64+15) << 35;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*45+11) = (uint64_t)IPW(ip, i*64+15) >> 29;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*45+11) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 61;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*45+12) = (uint64_t)IPW(ip, i*64+17) >> 3 | (uint64_t)IPX(ip, i*64+18) << 42;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*45+13) = (uint64_t)IPW(ip, i*64+18) >> 22 | (uint64_t)IPX(ip, i*64+19) << 23;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*45+14) = (uint64_t)IPW(ip, i*64+19) >> 41;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*45+14) |= (uint64_t)IPV(ip, i*64+20) << 4 | (uint64_t)IPX(ip, i*64+21) << 49;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*45+15) = (uint64_t)IPW(ip, i*64+21) >> 15 | (uint64_t)IPX(ip, i*64+22) << 30;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*45+16) = (uint64_t)IPW(ip, i*64+22) >> 34;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*45+16) |= (uint64_t)IPV(ip, i*64+23) << 11 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*45+17) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 37;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*45+18) = (uint64_t)IPW(ip, i*64+25) >> 27;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*45+18) |= (uint64_t)IPV(ip, i*64+26) << 18 | (uint64_t)IPX(ip, i*64+27) << 63;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*45+19) = (uint64_t)IPW(ip, i*64+27) >> 1 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*45+20) = (uint64_t)IPW(ip, i*64+28) >> 20 | (uint64_t)IPX(ip, i*64+29) << 25;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*45+21) = (uint64_t)IPW(ip, i*64+29) >> 39;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*45+21) |= (uint64_t)IPV(ip, i*64+30) << 6 | (uint64_t)IPX(ip, i*64+31) << 51;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*45+22) = (uint64_t)IPW(ip, i*64+31) >> 13;\
-}
-
-#define BITPACK64_45(ip, op, parm) { \
- BITBLK64_45(ip, 0, op, parm); IPI(ip); op += 45*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_46(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*23+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 46;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*23+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 18 | (uint64_t)IPX(ip, i*32+2) << 28;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*23+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 36;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*32+ 3) << 10 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*23+ 3) = (uint64_t)IPW(ip, i*32+ 4) >> 8 | (uint64_t)IPX(ip, i*32+5) << 38;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*23+ 4) = (uint64_t)IPW(ip, i*32+ 5) >> 26 | (uint64_t)IPX(ip, i*32+6) << 20;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*23+ 5) = (uint64_t)IPW(ip, i*32+ 6) >> 44;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*32+ 7) << 2 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*23+ 6) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 30;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*23+ 7) = (uint64_t)IPW(ip, i*32+ 9) >> 34;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*32+10) << 12 | (uint64_t)IPX(ip, i*32+11) << 58;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*23+ 8) = (uint64_t)IPW(ip, i*32+11) >> 6 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*23+ 9) = (uint64_t)IPW(ip, i*32+12) >> 24 | (uint64_t)IPX(ip, i*32+13) << 22;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*23+10) = (uint64_t)IPW(ip, i*32+13) >> 42;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*32+14) << 4 | (uint64_t)IPX(ip, i*32+15) << 50;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*23+11) = (uint64_t)IPW(ip, i*32+15) >> 14 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*23+12) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*23+12) |= (uint64_t)IPV(ip, i*32+17) << 14 | (uint64_t)IPX(ip, i*32+18) << 60;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*23+13) = (uint64_t)IPW(ip, i*32+18) >> 4 | (uint64_t)IPX(ip, i*32+19) << 42;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*23+14) = (uint64_t)IPW(ip, i*32+19) >> 22 | (uint64_t)IPX(ip, i*32+20) << 24;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*23+15) = (uint64_t)IPW(ip, i*32+20) >> 40;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*23+15) |= (uint64_t)IPV(ip, i*32+21) << 6 | (uint64_t)IPX(ip, i*32+22) << 52;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*23+16) = (uint64_t)IPW(ip, i*32+22) >> 12 | (uint64_t)IPX(ip, i*32+23) << 34;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*23+17) = (uint64_t)IPW(ip, i*32+23) >> 30;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*23+17) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 62;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*23+18) = (uint64_t)IPW(ip, i*32+25) >> 2 | (uint64_t)IPX(ip, i*32+26) << 44;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*23+19) = (uint64_t)IPW(ip, i*32+26) >> 20 | (uint64_t)IPX(ip, i*32+27) << 26;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*23+20) = (uint64_t)IPW(ip, i*32+27) >> 38;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*23+20) |= (uint64_t)IPV(ip, i*32+28) << 8 | (uint64_t)IPX(ip, i*32+29) << 54;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*23+21) = (uint64_t)IPW(ip, i*32+29) >> 10 | (uint64_t)IPX(ip, i*32+30) << 36;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*23+22) = (uint64_t)IPW(ip, i*32+30) >> 28;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*23+22) |= (uint64_t)IPV(ip, i*32+31) << 18;\
-}
-
-#define BITPACK64_46(ip, op, parm) { \
- BITBLK64_46(ip, 0, op, parm); IPI(ip); op += 46*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_47(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*47+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 47;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*47+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 17 | (uint64_t)IPX(ip, i*64+2) << 30;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*47+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 34;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*47+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 13 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*47+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 4 | (uint64_t)IPX(ip, i*64+5) << 43;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*47+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 21 | (uint64_t)IPX(ip, i*64+6) << 26;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*47+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 38;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*47+ 5) |= (uint64_t)IPV(ip, i*64+ 7) << 9 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*47+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 39;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*47+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 25 | (uint64_t)IPX(ip, i*64+10) << 22;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*47+ 8) = (uint64_t)IPW(ip, i*64+10) >> 42;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*47+ 8) |= (uint64_t)IPV(ip, i*64+11) << 5 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*47+ 9) = (uint64_t)IPW(ip, i*64+12) >> 12 | (uint64_t)IPX(ip, i*64+13) << 35;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*47+10) = (uint64_t)IPW(ip, i*64+13) >> 29 | (uint64_t)IPX(ip, i*64+14) << 18;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*47+11) = (uint64_t)IPW(ip, i*64+14) >> 46;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*47+11) |= (uint64_t)IPV(ip, i*64+15) << 1 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*47+12) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 31;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*47+13) = (uint64_t)IPW(ip, i*64+17) >> 33;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*47+13) |= (uint64_t)IPV(ip, i*64+18) << 14 | (uint64_t)IPX(ip, i*64+19) << 61;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*47+14) = (uint64_t)IPW(ip, i*64+19) >> 3 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*47+15) = (uint64_t)IPW(ip, i*64+20) >> 20 | (uint64_t)IPX(ip, i*64+21) << 27;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*47+16) = (uint64_t)IPW(ip, i*64+21) >> 37;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*47+16) |= (uint64_t)IPV(ip, i*64+22) << 10 | (uint64_t)IPX(ip, i*64+23) << 57;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*47+17) = (uint64_t)IPW(ip, i*64+23) >> 7 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*47+18) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 23;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*47+19) = (uint64_t)IPW(ip, i*64+25) >> 41;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*47+19) |= (uint64_t)IPV(ip, i*64+26) << 6 | (uint64_t)IPX(ip, i*64+27) << 53;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*47+20) = (uint64_t)IPW(ip, i*64+27) >> 11 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*47+21) = (uint64_t)IPW(ip, i*64+28) >> 28 | (uint64_t)IPX(ip, i*64+29) << 19;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*47+22) = (uint64_t)IPW(ip, i*64+29) >> 45;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*47+22) |= (uint64_t)IPV(ip, i*64+30) << 2 | (uint64_t)IPX(ip, i*64+31) << 49;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*47+23) = (uint64_t)IPW(ip, i*64+31) >> 15;\
-}
-
-#define BITPACK64_47(ip, op, parm) { \
- BITBLK64_47(ip, 0, op, parm); IPI(ip); op += 47*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_48(ip, i, op, parm) { ;\
- IP9(ip, i*4+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*4+ 0) | (uint64_t)IPX(ip, i*4+1) << 48;\
- IP64(ip, i*4+ 1, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*4+ 1) >> 16 | (uint64_t)IPX(ip, i*4+2) << 32;\
- IP64(ip, i*4+ 2, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*4+ 2) >> 32;\
- IP9(ip, i*4+ 3, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*4+ 3) << 16;\
-}
-
-#define BITPACK64_48(ip, op, parm) { \
- BITBLK64_48(ip, 0, op, parm);\
- BITBLK64_48(ip, 1, op, parm);\
- BITBLK64_48(ip, 2, op, parm);\
- BITBLK64_48(ip, 3, op, parm);\
- BITBLK64_48(ip, 4, op, parm);\
- BITBLK64_48(ip, 5, op, parm);\
- BITBLK64_48(ip, 6, op, parm);\
- BITBLK64_48(ip, 7, op, parm); IPI(ip); op += 48*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_49(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*49+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 49;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*49+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 15 | (uint64_t)IPX(ip, i*64+2) << 34;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*49+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 30 | (uint64_t)IPX(ip, i*64+3) << 19;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*49+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 45;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*49+ 3) |= (uint64_t)IPV(ip, i*64+ 4) << 4 | (uint64_t)IPX(ip, i*64+5) << 53;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*49+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 11 | (uint64_t)IPX(ip, i*64+6) << 38;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*49+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 26 | (uint64_t)IPX(ip, i*64+7) << 23;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*49+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 41;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*49+ 6) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 57;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*49+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 7 | (uint64_t)IPX(ip, i*64+10) << 42;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*49+ 8) = (uint64_t)IPW(ip, i*64+10) >> 22 | (uint64_t)IPX(ip, i*64+11) << 27;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*49+ 9) = (uint64_t)IPW(ip, i*64+11) >> 37;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*49+ 9) |= (uint64_t)IPV(ip, i*64+12) << 12 | (uint64_t)IPX(ip, i*64+13) << 61;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*49+10) = (uint64_t)IPW(ip, i*64+13) >> 3 | (uint64_t)IPX(ip, i*64+14) << 46;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*49+11) = (uint64_t)IPW(ip, i*64+14) >> 18 | (uint64_t)IPX(ip, i*64+15) << 31;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*49+12) = (uint64_t)IPW(ip, i*64+15) >> 33 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*49+13) = (uint64_t)IPW(ip, i*64+16) >> 48;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*49+13) |= (uint64_t)IPV(ip, i*64+17) << 1 | (uint64_t)IPX(ip, i*64+18) << 50;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*49+14) = (uint64_t)IPW(ip, i*64+18) >> 14 | (uint64_t)IPX(ip, i*64+19) << 35;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*49+15) = (uint64_t)IPW(ip, i*64+19) >> 29 | (uint64_t)IPX(ip, i*64+20) << 20;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*49+16) = (uint64_t)IPW(ip, i*64+20) >> 44;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*49+16) |= (uint64_t)IPV(ip, i*64+21) << 5 | (uint64_t)IPX(ip, i*64+22) << 54;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*49+17) = (uint64_t)IPW(ip, i*64+22) >> 10 | (uint64_t)IPX(ip, i*64+23) << 39;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*49+18) = (uint64_t)IPW(ip, i*64+23) >> 25 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*49+19) = (uint64_t)IPW(ip, i*64+24) >> 40;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*49+19) |= (uint64_t)IPV(ip, i*64+25) << 9 | (uint64_t)IPX(ip, i*64+26) << 58;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*49+20) = (uint64_t)IPW(ip, i*64+26) >> 6 | (uint64_t)IPX(ip, i*64+27) << 43;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*49+21) = (uint64_t)IPW(ip, i*64+27) >> 21 | (uint64_t)IPX(ip, i*64+28) << 28;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*49+22) = (uint64_t)IPW(ip, i*64+28) >> 36;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*49+22) |= (uint64_t)IPV(ip, i*64+29) << 13 | (uint64_t)IPX(ip, i*64+30) << 62;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*49+23) = (uint64_t)IPW(ip, i*64+30) >> 2 | (uint64_t)IPX(ip, i*64+31) << 47;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*49+24) = (uint64_t)IPW(ip, i*64+31) >> 17;\
-}
-
-#define BITPACK64_49(ip, op, parm) { \
- BITBLK64_49(ip, 0, op, parm); IPI(ip); op += 49*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_50(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*25+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 50;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*25+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 14 | (uint64_t)IPX(ip, i*32+2) << 36;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*25+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 28 | (uint64_t)IPX(ip, i*32+3) << 22;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*25+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 42;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*32+ 4) << 8 | (uint64_t)IPX(ip, i*32+5) << 58;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*25+ 4) = (uint64_t)IPW(ip, i*32+ 5) >> 6 | (uint64_t)IPX(ip, i*32+6) << 44;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*25+ 5) = (uint64_t)IPW(ip, i*32+ 6) >> 20 | (uint64_t)IPX(ip, i*32+7) << 30;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*25+ 6) = (uint64_t)IPW(ip, i*32+ 7) >> 34 | (uint64_t)IPX(ip, i*32+8) << 16;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*25+ 7) = (uint64_t)IPW(ip, i*32+ 8) >> 48;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*32+ 9) << 2 | (uint64_t)IPX(ip, i*32+10) << 52;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*25+ 8) = (uint64_t)IPW(ip, i*32+10) >> 12 | (uint64_t)IPX(ip, i*32+11) << 38;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*25+ 9) = (uint64_t)IPW(ip, i*32+11) >> 26 | (uint64_t)IPX(ip, i*32+12) << 24;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*25+10) = (uint64_t)IPW(ip, i*32+12) >> 40;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*32+13) << 10 | (uint64_t)IPX(ip, i*32+14) << 60;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*25+11) = (uint64_t)IPW(ip, i*32+14) >> 4 | (uint64_t)IPX(ip, i*32+15) << 46;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*25+12) = (uint64_t)IPW(ip, i*32+15) >> 18 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*25+13) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 18;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*25+14) = (uint64_t)IPW(ip, i*32+17) >> 46;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*25+14) |= (uint64_t)IPV(ip, i*32+18) << 4 | (uint64_t)IPX(ip, i*32+19) << 54;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*25+15) = (uint64_t)IPW(ip, i*32+19) >> 10 | (uint64_t)IPX(ip, i*32+20) << 40;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*25+16) = (uint64_t)IPW(ip, i*32+20) >> 24 | (uint64_t)IPX(ip, i*32+21) << 26;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*25+17) = (uint64_t)IPW(ip, i*32+21) >> 38;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*25+17) |= (uint64_t)IPV(ip, i*32+22) << 12 | (uint64_t)IPX(ip, i*32+23) << 62;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*25+18) = (uint64_t)IPW(ip, i*32+23) >> 2 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*25+19) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 34;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*25+20) = (uint64_t)IPW(ip, i*32+25) >> 30 | (uint64_t)IPX(ip, i*32+26) << 20;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*25+21) = (uint64_t)IPW(ip, i*32+26) >> 44;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*25+21) |= (uint64_t)IPV(ip, i*32+27) << 6 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*25+22) = (uint64_t)IPW(ip, i*32+28) >> 8 | (uint64_t)IPX(ip, i*32+29) << 42;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*25+23) = (uint64_t)IPW(ip, i*32+29) >> 22 | (uint64_t)IPX(ip, i*32+30) << 28;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*25+24) = (uint64_t)IPW(ip, i*32+30) >> 36;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*25+24) |= (uint64_t)IPV(ip, i*32+31) << 14;\
-}
-
-#define BITPACK64_50(ip, op, parm) { \
- BITBLK64_50(ip, 0, op, parm); IPI(ip); op += 50*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_51(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*51+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 51;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*51+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 13 | (uint64_t)IPX(ip, i*64+2) << 38;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*51+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 26 | (uint64_t)IPX(ip, i*64+3) << 25;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*51+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 39;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*51+ 3) |= (uint64_t)IPV(ip, i*64+ 4) << 12 | (uint64_t)IPX(ip, i*64+5) << 63;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*51+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 1 | (uint64_t)IPX(ip, i*64+6) << 50;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*51+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 14 | (uint64_t)IPX(ip, i*64+7) << 37;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*51+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 27 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*51+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 40;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*51+ 7) |= (uint64_t)IPV(ip, i*64+ 9) << 11 | (uint64_t)IPX(ip, i*64+10) << 62;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*51+ 8) = (uint64_t)IPW(ip, i*64+10) >> 2 | (uint64_t)IPX(ip, i*64+11) << 49;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*51+ 9) = (uint64_t)IPW(ip, i*64+11) >> 15 | (uint64_t)IPX(ip, i*64+12) << 36;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*51+10) = (uint64_t)IPW(ip, i*64+12) >> 28 | (uint64_t)IPX(ip, i*64+13) << 23;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*51+11) = (uint64_t)IPW(ip, i*64+13) >> 41;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*51+11) |= (uint64_t)IPV(ip, i*64+14) << 10 | (uint64_t)IPX(ip, i*64+15) << 61;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*51+12) = (uint64_t)IPW(ip, i*64+15) >> 3 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*51+13) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 35;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*51+14) = (uint64_t)IPW(ip, i*64+17) >> 29 | (uint64_t)IPX(ip, i*64+18) << 22;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*51+15) = (uint64_t)IPW(ip, i*64+18) >> 42;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*51+15) |= (uint64_t)IPV(ip, i*64+19) << 9 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*51+16) = (uint64_t)IPW(ip, i*64+20) >> 4 | (uint64_t)IPX(ip, i*64+21) << 47;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*51+17) = (uint64_t)IPW(ip, i*64+21) >> 17 | (uint64_t)IPX(ip, i*64+22) << 34;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*51+18) = (uint64_t)IPW(ip, i*64+22) >> 30 | (uint64_t)IPX(ip, i*64+23) << 21;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*51+19) = (uint64_t)IPW(ip, i*64+23) >> 43;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*51+19) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 59;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*51+20) = (uint64_t)IPW(ip, i*64+25) >> 5 | (uint64_t)IPX(ip, i*64+26) << 46;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*51+21) = (uint64_t)IPW(ip, i*64+26) >> 18 | (uint64_t)IPX(ip, i*64+27) << 33;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*51+22) = (uint64_t)IPW(ip, i*64+27) >> 31 | (uint64_t)IPX(ip, i*64+28) << 20;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*51+23) = (uint64_t)IPW(ip, i*64+28) >> 44;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*51+23) |= (uint64_t)IPV(ip, i*64+29) << 7 | (uint64_t)IPX(ip, i*64+30) << 58;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*51+24) = (uint64_t)IPW(ip, i*64+30) >> 6 | (uint64_t)IPX(ip, i*64+31) << 45;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*51+25) = (uint64_t)IPW(ip, i*64+31) >> 19;\
-}
-
-#define BITPACK64_51(ip, op, parm) { \
- BITBLK64_51(ip, 0, op, parm); IPI(ip); op += 51*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_52(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 52;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 12 | (uint64_t)IPX(ip, i*16+2) << 40;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 24 | (uint64_t)IPX(ip, i*16+3) << 28;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*16+ 3) >> 36 | (uint64_t)IPX(ip, i*16+4) << 16;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*16+ 4) >> 48;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*16+ 5) << 4 | (uint64_t)IPX(ip, i*16+6) << 56;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*16+ 6) >> 8 | (uint64_t)IPX(ip, i*16+7) << 44;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*16+ 7) >> 20 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*13+ 7) = (uint64_t)IPW(ip, i*16+ 8) >> 32 | (uint64_t)IPX(ip, i*16+9) << 20;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*13+ 8) = (uint64_t)IPW(ip, i*16+ 9) >> 44;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*16+10) << 8 | (uint64_t)IPX(ip, i*16+11) << 60;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*13+ 9) = (uint64_t)IPW(ip, i*16+11) >> 4 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*13+10) = (uint64_t)IPW(ip, i*16+12) >> 16 | (uint64_t)IPX(ip, i*16+13) << 36;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*13+11) = (uint64_t)IPW(ip, i*16+13) >> 28 | (uint64_t)IPX(ip, i*16+14) << 24;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*13+12) = (uint64_t)IPW(ip, i*16+14) >> 40;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*16+15) << 12;\
-}
-
-#define BITPACK64_52(ip, op, parm) { \
- BITBLK64_52(ip, 0, op, parm);\
- BITBLK64_52(ip, 1, op, parm); IPI(ip); op += 52*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_53(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*53+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 53;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*53+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 11 | (uint64_t)IPX(ip, i*64+2) << 42;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*53+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 22 | (uint64_t)IPX(ip, i*64+3) << 31;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*53+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 33 | (uint64_t)IPX(ip, i*64+4) << 20;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*53+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 44;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*53+ 4) |= (uint64_t)IPV(ip, i*64+ 5) << 9 | (uint64_t)IPX(ip, i*64+6) << 62;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*53+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 2 | (uint64_t)IPX(ip, i*64+7) << 51;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*53+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 13 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*53+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 29;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*53+ 8) = (uint64_t)IPW(ip, i*64+ 9) >> 35 | (uint64_t)IPX(ip, i*64+10) << 18;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*53+ 9) = (uint64_t)IPW(ip, i*64+10) >> 46;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*53+ 9) |= (uint64_t)IPV(ip, i*64+11) << 7 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*53+10) = (uint64_t)IPW(ip, i*64+12) >> 4 | (uint64_t)IPX(ip, i*64+13) << 49;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*53+11) = (uint64_t)IPW(ip, i*64+13) >> 15 | (uint64_t)IPX(ip, i*64+14) << 38;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*53+12) = (uint64_t)IPW(ip, i*64+14) >> 26 | (uint64_t)IPX(ip, i*64+15) << 27;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*53+13) = (uint64_t)IPW(ip, i*64+15) >> 37 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*53+14) = (uint64_t)IPW(ip, i*64+16) >> 48;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*53+14) |= (uint64_t)IPV(ip, i*64+17) << 5 | (uint64_t)IPX(ip, i*64+18) << 58;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*53+15) = (uint64_t)IPW(ip, i*64+18) >> 6 | (uint64_t)IPX(ip, i*64+19) << 47;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*53+16) = (uint64_t)IPW(ip, i*64+19) >> 17 | (uint64_t)IPX(ip, i*64+20) << 36;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*53+17) = (uint64_t)IPW(ip, i*64+20) >> 28 | (uint64_t)IPX(ip, i*64+21) << 25;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*53+18) = (uint64_t)IPW(ip, i*64+21) >> 39 | (uint64_t)IPX(ip, i*64+22) << 14;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*53+19) = (uint64_t)IPW(ip, i*64+22) >> 50;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*53+19) |= (uint64_t)IPV(ip, i*64+23) << 3 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*53+20) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 45;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*53+21) = (uint64_t)IPW(ip, i*64+25) >> 19 | (uint64_t)IPX(ip, i*64+26) << 34;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*53+22) = (uint64_t)IPW(ip, i*64+26) >> 30 | (uint64_t)IPX(ip, i*64+27) << 23;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*53+23) = (uint64_t)IPW(ip, i*64+27) >> 41 | (uint64_t)IPX(ip, i*64+28) << 12;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*53+24) = (uint64_t)IPW(ip, i*64+28) >> 52;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*53+24) |= (uint64_t)IPV(ip, i*64+29) << 1 | (uint64_t)IPX(ip, i*64+30) << 54;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*53+25) = (uint64_t)IPW(ip, i*64+30) >> 10 | (uint64_t)IPX(ip, i*64+31) << 43;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*53+26) = (uint64_t)IPW(ip, i*64+31) >> 21;\
-}
-
-#define BITPACK64_53(ip, op, parm) { \
- BITBLK64_53(ip, 0, op, parm); IPI(ip); op += 53*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_54(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*27+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 54;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*27+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 10 | (uint64_t)IPX(ip, i*32+2) << 44;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*27+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 20 | (uint64_t)IPX(ip, i*32+3) << 34;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*27+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 30 | (uint64_t)IPX(ip, i*32+4) << 24;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*27+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 40 | (uint64_t)IPX(ip, i*32+5) << 14;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*27+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 50;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*32+ 6) << 4 | (uint64_t)IPX(ip, i*32+7) << 58;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*27+ 6) = (uint64_t)IPW(ip, i*32+ 7) >> 6 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*27+ 7) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 38;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*27+ 8) = (uint64_t)IPW(ip, i*32+ 9) >> 26 | (uint64_t)IPX(ip, i*32+10) << 28;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*27+ 9) = (uint64_t)IPW(ip, i*32+10) >> 36 | (uint64_t)IPX(ip, i*32+11) << 18;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*27+10) = (uint64_t)IPW(ip, i*32+11) >> 46;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*32+12) << 8 | (uint64_t)IPX(ip, i*32+13) << 62;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*27+11) = (uint64_t)IPW(ip, i*32+13) >> 2 | (uint64_t)IPX(ip, i*32+14) << 52;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*27+12) = (uint64_t)IPW(ip, i*32+14) >> 12 | (uint64_t)IPX(ip, i*32+15) << 42;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*27+13) = (uint64_t)IPW(ip, i*32+15) >> 22 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*27+14) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 22;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*27+15) = (uint64_t)IPW(ip, i*32+17) >> 42 | (uint64_t)IPX(ip, i*32+18) << 12;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*27+16) = (uint64_t)IPW(ip, i*32+18) >> 52;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*27+16) |= (uint64_t)IPV(ip, i*32+19) << 2 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*27+17) = (uint64_t)IPW(ip, i*32+20) >> 8 | (uint64_t)IPX(ip, i*32+21) << 46;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*27+18) = (uint64_t)IPW(ip, i*32+21) >> 18 | (uint64_t)IPX(ip, i*32+22) << 36;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*27+19) = (uint64_t)IPW(ip, i*32+22) >> 28 | (uint64_t)IPX(ip, i*32+23) << 26;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*27+20) = (uint64_t)IPW(ip, i*32+23) >> 38 | (uint64_t)IPX(ip, i*32+24) << 16;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*27+21) = (uint64_t)IPW(ip, i*32+24) >> 48;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*27+21) |= (uint64_t)IPV(ip, i*32+25) << 6 | (uint64_t)IPX(ip, i*32+26) << 60;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*27+22) = (uint64_t)IPW(ip, i*32+26) >> 4 | (uint64_t)IPX(ip, i*32+27) << 50;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*27+23) = (uint64_t)IPW(ip, i*32+27) >> 14 | (uint64_t)IPX(ip, i*32+28) << 40;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*27+24) = (uint64_t)IPW(ip, i*32+28) >> 24 | (uint64_t)IPX(ip, i*32+29) << 30;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*27+25) = (uint64_t)IPW(ip, i*32+29) >> 34 | (uint64_t)IPX(ip, i*32+30) << 20;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*27+26) = (uint64_t)IPW(ip, i*32+30) >> 44;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*27+26) |= (uint64_t)IPV(ip, i*32+31) << 10;\
-}
-
-#define BITPACK64_54(ip, op, parm) { \
- BITBLK64_54(ip, 0, op, parm); IPI(ip); op += 54*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_55(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*55+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 55;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*55+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 9 | (uint64_t)IPX(ip, i*64+2) << 46;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*55+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 18 | (uint64_t)IPX(ip, i*64+3) << 37;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*55+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 27 | (uint64_t)IPX(ip, i*64+4) << 28;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*55+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 36 | (uint64_t)IPX(ip, i*64+5) << 19;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*55+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 45 | (uint64_t)IPX(ip, i*64+6) << 10;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*55+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 54;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*55+ 6) |= (uint64_t)IPV(ip, i*64+ 7) << 1 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*55+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 47;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*55+ 8) = (uint64_t)IPW(ip, i*64+ 9) >> 17 | (uint64_t)IPX(ip, i*64+10) << 38;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*55+ 9) = (uint64_t)IPW(ip, i*64+10) >> 26 | (uint64_t)IPX(ip, i*64+11) << 29;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*55+10) = (uint64_t)IPW(ip, i*64+11) >> 35 | (uint64_t)IPX(ip, i*64+12) << 20;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*55+11) = (uint64_t)IPW(ip, i*64+12) >> 44 | (uint64_t)IPX(ip, i*64+13) << 11;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*55+12) = (uint64_t)IPW(ip, i*64+13) >> 53;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*55+12) |= (uint64_t)IPV(ip, i*64+14) << 2 | (uint64_t)IPX(ip, i*64+15) << 57;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*55+13) = (uint64_t)IPW(ip, i*64+15) >> 7 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*55+14) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 39;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*55+15) = (uint64_t)IPW(ip, i*64+17) >> 25 | (uint64_t)IPX(ip, i*64+18) << 30;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*55+16) = (uint64_t)IPW(ip, i*64+18) >> 34 | (uint64_t)IPX(ip, i*64+19) << 21;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*55+17) = (uint64_t)IPW(ip, i*64+19) >> 43 | (uint64_t)IPX(ip, i*64+20) << 12;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*55+18) = (uint64_t)IPW(ip, i*64+20) >> 52;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*55+18) |= (uint64_t)IPV(ip, i*64+21) << 3 | (uint64_t)IPX(ip, i*64+22) << 58;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*55+19) = (uint64_t)IPW(ip, i*64+22) >> 6 | (uint64_t)IPX(ip, i*64+23) << 49;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*55+20) = (uint64_t)IPW(ip, i*64+23) >> 15 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*55+21) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 31;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*55+22) = (uint64_t)IPW(ip, i*64+25) >> 33 | (uint64_t)IPX(ip, i*64+26) << 22;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*55+23) = (uint64_t)IPW(ip, i*64+26) >> 42 | (uint64_t)IPX(ip, i*64+27) << 13;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*55+24) = (uint64_t)IPW(ip, i*64+27) >> 51;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*55+24) |= (uint64_t)IPV(ip, i*64+28) << 4 | (uint64_t)IPX(ip, i*64+29) << 59;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*55+25) = (uint64_t)IPW(ip, i*64+29) >> 5 | (uint64_t)IPX(ip, i*64+30) << 50;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*55+26) = (uint64_t)IPW(ip, i*64+30) >> 14 | (uint64_t)IPX(ip, i*64+31) << 41;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*55+27) = (uint64_t)IPW(ip, i*64+31) >> 23;\
-}
-
-#define BITPACK64_55(ip, op, parm) { \
- BITBLK64_55(ip, 0, op, parm); IPI(ip); op += 55*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_56(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*8+ 0) | (uint64_t)IPX(ip, i*8+1) << 56;\
- IP64(ip, i*8+ 1, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*8+ 1) >> 8 | (uint64_t)IPX(ip, i*8+2) << 48;\
- IP64(ip, i*8+ 2, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*8+ 2) >> 16 | (uint64_t)IPX(ip, i*8+3) << 40;\
- IP64(ip, i*8+ 3, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*8+ 3) >> 24 | (uint64_t)IPX(ip, i*8+4) << 32;\
- IP64(ip, i*8+ 4, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*8+ 4) >> 32 | (uint64_t)IPX(ip, i*8+5) << 24;\
- IP64(ip, i*8+ 5, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*8+ 5) >> 40 | (uint64_t)IPX(ip, i*8+6) << 16;\
- IP64(ip, i*8+ 6, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*8+ 6) >> 48;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*8+ 7) << 8;\
-}
-
-#define BITPACK64_56(ip, op, parm) { \
- BITBLK64_56(ip, 0, op, parm);\
- BITBLK64_56(ip, 1, op, parm);\
- BITBLK64_56(ip, 2, op, parm);\
- BITBLK64_56(ip, 3, op, parm); IPI(ip); op += 56*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_57(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*57+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 57;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*57+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 7 | (uint64_t)IPX(ip, i*64+2) << 50;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*57+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 14 | (uint64_t)IPX(ip, i*64+3) << 43;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*57+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 21 | (uint64_t)IPX(ip, i*64+4) << 36;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*57+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 28 | (uint64_t)IPX(ip, i*64+5) << 29;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*57+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 35 | (uint64_t)IPX(ip, i*64+6) << 22;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*57+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 42 | (uint64_t)IPX(ip, i*64+7) << 15;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*57+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 49 | (uint64_t)IPX(ip, i*64+8) << 8;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*57+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 56;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*57+ 8) |= (uint64_t)IPV(ip, i*64+ 9) << 1 | (uint64_t)IPX(ip, i*64+10) << 58;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*57+ 9) = (uint64_t)IPW(ip, i*64+10) >> 6 | (uint64_t)IPX(ip, i*64+11) << 51;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*57+10) = (uint64_t)IPW(ip, i*64+11) >> 13 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*57+11) = (uint64_t)IPW(ip, i*64+12) >> 20 | (uint64_t)IPX(ip, i*64+13) << 37;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*57+12) = (uint64_t)IPW(ip, i*64+13) >> 27 | (uint64_t)IPX(ip, i*64+14) << 30;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*57+13) = (uint64_t)IPW(ip, i*64+14) >> 34 | (uint64_t)IPX(ip, i*64+15) << 23;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*57+14) = (uint64_t)IPW(ip, i*64+15) >> 41 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*57+15) = (uint64_t)IPW(ip, i*64+16) >> 48 | (uint64_t)IPX(ip, i*64+17) << 9;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*57+16) = (uint64_t)IPW(ip, i*64+17) >> 55;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*57+16) |= (uint64_t)IPV(ip, i*64+18) << 2 | (uint64_t)IPX(ip, i*64+19) << 59;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*57+17) = (uint64_t)IPW(ip, i*64+19) >> 5 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*57+18) = (uint64_t)IPW(ip, i*64+20) >> 12 | (uint64_t)IPX(ip, i*64+21) << 45;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*57+19) = (uint64_t)IPW(ip, i*64+21) >> 19 | (uint64_t)IPX(ip, i*64+22) << 38;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*57+20) = (uint64_t)IPW(ip, i*64+22) >> 26 | (uint64_t)IPX(ip, i*64+23) << 31;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*57+21) = (uint64_t)IPW(ip, i*64+23) >> 33 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*57+22) = (uint64_t)IPW(ip, i*64+24) >> 40 | (uint64_t)IPX(ip, i*64+25) << 17;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*57+23) = (uint64_t)IPW(ip, i*64+25) >> 47 | (uint64_t)IPX(ip, i*64+26) << 10;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*57+24) = (uint64_t)IPW(ip, i*64+26) >> 54;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*57+24) |= (uint64_t)IPV(ip, i*64+27) << 3 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*57+25) = (uint64_t)IPW(ip, i*64+28) >> 4 | (uint64_t)IPX(ip, i*64+29) << 53;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*57+26) = (uint64_t)IPW(ip, i*64+29) >> 11 | (uint64_t)IPX(ip, i*64+30) << 46;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*57+27) = (uint64_t)IPW(ip, i*64+30) >> 18 | (uint64_t)IPX(ip, i*64+31) << 39;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*57+28) = (uint64_t)IPW(ip, i*64+31) >> 25;\
-}
-
-#define BITPACK64_57(ip, op, parm) { \
- BITBLK64_57(ip, 0, op, parm); IPI(ip); op += 57*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_58(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*29+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 58;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*29+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 6 | (uint64_t)IPX(ip, i*32+2) << 52;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*29+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 12 | (uint64_t)IPX(ip, i*32+3) << 46;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*29+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 18 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*29+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 24 | (uint64_t)IPX(ip, i*32+5) << 34;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*29+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 30 | (uint64_t)IPX(ip, i*32+6) << 28;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*29+ 6) = (uint64_t)IPW(ip, i*32+ 6) >> 36 | (uint64_t)IPX(ip, i*32+7) << 22;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*29+ 7) = (uint64_t)IPW(ip, i*32+ 7) >> 42 | (uint64_t)IPX(ip, i*32+8) << 16;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*29+ 8) = (uint64_t)IPW(ip, i*32+ 8) >> 48 | (uint64_t)IPX(ip, i*32+9) << 10;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*29+ 9) = (uint64_t)IPW(ip, i*32+ 9) >> 54;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*32+10) << 4 | (uint64_t)IPX(ip, i*32+11) << 62;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*29+10) = (uint64_t)IPW(ip, i*32+11) >> 2 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*29+11) = (uint64_t)IPW(ip, i*32+12) >> 8 | (uint64_t)IPX(ip, i*32+13) << 50;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*29+12) = (uint64_t)IPW(ip, i*32+13) >> 14 | (uint64_t)IPX(ip, i*32+14) << 44;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*29+13) = (uint64_t)IPW(ip, i*32+14) >> 20 | (uint64_t)IPX(ip, i*32+15) << 38;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*29+14) = (uint64_t)IPW(ip, i*32+15) >> 26 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*29+15) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 26;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*29+16) = (uint64_t)IPW(ip, i*32+17) >> 38 | (uint64_t)IPX(ip, i*32+18) << 20;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*29+17) = (uint64_t)IPW(ip, i*32+18) >> 44 | (uint64_t)IPX(ip, i*32+19) << 14;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*29+18) = (uint64_t)IPW(ip, i*32+19) >> 50 | (uint64_t)IPX(ip, i*32+20) << 8;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*29+19) = (uint64_t)IPW(ip, i*32+20) >> 56;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*29+19) |= (uint64_t)IPV(ip, i*32+21) << 2 | (uint64_t)IPX(ip, i*32+22) << 60;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*29+20) = (uint64_t)IPW(ip, i*32+22) >> 4 | (uint64_t)IPX(ip, i*32+23) << 54;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*29+21) = (uint64_t)IPW(ip, i*32+23) >> 10 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*29+22) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 42;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*29+23) = (uint64_t)IPW(ip, i*32+25) >> 22 | (uint64_t)IPX(ip, i*32+26) << 36;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*29+24) = (uint64_t)IPW(ip, i*32+26) >> 28 | (uint64_t)IPX(ip, i*32+27) << 30;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*29+25) = (uint64_t)IPW(ip, i*32+27) >> 34 | (uint64_t)IPX(ip, i*32+28) << 24;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*29+26) = (uint64_t)IPW(ip, i*32+28) >> 40 | (uint64_t)IPX(ip, i*32+29) << 18;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*29+27) = (uint64_t)IPW(ip, i*32+29) >> 46 | (uint64_t)IPX(ip, i*32+30) << 12;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*29+28) = (uint64_t)IPW(ip, i*32+30) >> 52;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*29+28) |= (uint64_t)IPV(ip, i*32+31) << 6;\
-}
-
-#define BITPACK64_58(ip, op, parm) { \
- BITBLK64_58(ip, 0, op, parm); IPI(ip); op += 58*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_59(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*59+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 59;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*59+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 5 | (uint64_t)IPX(ip, i*64+2) << 54;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*59+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 10 | (uint64_t)IPX(ip, i*64+3) << 49;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*59+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 15 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*59+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 20 | (uint64_t)IPX(ip, i*64+5) << 39;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*59+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 25 | (uint64_t)IPX(ip, i*64+6) << 34;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*59+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 30 | (uint64_t)IPX(ip, i*64+7) << 29;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*59+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 35 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*59+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 40 | (uint64_t)IPX(ip, i*64+9) << 19;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*59+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 45 | (uint64_t)IPX(ip, i*64+10) << 14;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*59+10) = (uint64_t)IPW(ip, i*64+10) >> 50 | (uint64_t)IPX(ip, i*64+11) << 9;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*59+11) = (uint64_t)IPW(ip, i*64+11) >> 55;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*59+11) |= (uint64_t)IPV(ip, i*64+12) << 4 | (uint64_t)IPX(ip, i*64+13) << 63;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*59+12) = (uint64_t)IPW(ip, i*64+13) >> 1 | (uint64_t)IPX(ip, i*64+14) << 58;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*59+13) = (uint64_t)IPW(ip, i*64+14) >> 6 | (uint64_t)IPX(ip, i*64+15) << 53;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*59+14) = (uint64_t)IPW(ip, i*64+15) >> 11 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*59+15) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 43;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*59+16) = (uint64_t)IPW(ip, i*64+17) >> 21 | (uint64_t)IPX(ip, i*64+18) << 38;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*59+17) = (uint64_t)IPW(ip, i*64+18) >> 26 | (uint64_t)IPX(ip, i*64+19) << 33;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*59+18) = (uint64_t)IPW(ip, i*64+19) >> 31 | (uint64_t)IPX(ip, i*64+20) << 28;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*59+19) = (uint64_t)IPW(ip, i*64+20) >> 36 | (uint64_t)IPX(ip, i*64+21) << 23;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*59+20) = (uint64_t)IPW(ip, i*64+21) >> 41 | (uint64_t)IPX(ip, i*64+22) << 18;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*59+21) = (uint64_t)IPW(ip, i*64+22) >> 46 | (uint64_t)IPX(ip, i*64+23) << 13;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*59+22) = (uint64_t)IPW(ip, i*64+23) >> 51 | (uint64_t)IPX(ip, i*64+24) << 8;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*59+23) = (uint64_t)IPW(ip, i*64+24) >> 56;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*59+23) |= (uint64_t)IPV(ip, i*64+25) << 3 | (uint64_t)IPX(ip, i*64+26) << 62;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*59+24) = (uint64_t)IPW(ip, i*64+26) >> 2 | (uint64_t)IPX(ip, i*64+27) << 57;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*59+25) = (uint64_t)IPW(ip, i*64+27) >> 7 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*59+26) = (uint64_t)IPW(ip, i*64+28) >> 12 | (uint64_t)IPX(ip, i*64+29) << 47;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*59+27) = (uint64_t)IPW(ip, i*64+29) >> 17 | (uint64_t)IPX(ip, i*64+30) << 42;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*59+28) = (uint64_t)IPW(ip, i*64+30) >> 22 | (uint64_t)IPX(ip, i*64+31) << 37;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*59+29) = (uint64_t)IPW(ip, i*64+31) >> 27;\
-}
-
-#define BITPACK64_59(ip, op, parm) { \
- BITBLK64_59(ip, 0, op, parm); IPI(ip); op += 59*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_60(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 60;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 4 | (uint64_t)IPX(ip, i*16+2) << 56;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 8 | (uint64_t)IPX(ip, i*16+3) << 52;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*16+ 3) >> 12 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*16+ 4) >> 16 | (uint64_t)IPX(ip, i*16+5) << 44;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*16+ 5) >> 20 | (uint64_t)IPX(ip, i*16+6) << 40;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*16+ 6) >> 24 | (uint64_t)IPX(ip, i*16+7) << 36;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*16+ 7) >> 28 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*15+ 8) = (uint64_t)IPW(ip, i*16+ 8) >> 32 | (uint64_t)IPX(ip, i*16+9) << 28;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*15+ 9) = (uint64_t)IPW(ip, i*16+ 9) >> 36 | (uint64_t)IPX(ip, i*16+10) << 24;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*15+10) = (uint64_t)IPW(ip, i*16+10) >> 40 | (uint64_t)IPX(ip, i*16+11) << 20;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*15+11) = (uint64_t)IPW(ip, i*16+11) >> 44 | (uint64_t)IPX(ip, i*16+12) << 16;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*15+12) = (uint64_t)IPW(ip, i*16+12) >> 48 | (uint64_t)IPX(ip, i*16+13) << 12;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*15+13) = (uint64_t)IPW(ip, i*16+13) >> 52 | (uint64_t)IPX(ip, i*16+14) << 8;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*15+14) = (uint64_t)IPW(ip, i*16+14) >> 56;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*16+15) << 4;\
-}
-
-#define BITPACK64_60(ip, op, parm) { \
- BITBLK64_60(ip, 0, op, parm);\
- BITBLK64_60(ip, 1, op, parm); IPI(ip); op += 60*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_61(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*61+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 61;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*61+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 3 | (uint64_t)IPX(ip, i*64+2) << 58;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*61+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 6 | (uint64_t)IPX(ip, i*64+3) << 55;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*61+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 9 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*61+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 12 | (uint64_t)IPX(ip, i*64+5) << 49;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*61+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 15 | (uint64_t)IPX(ip, i*64+6) << 46;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*61+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 18 | (uint64_t)IPX(ip, i*64+7) << 43;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*61+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 21 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*61+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 37;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*61+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 27 | (uint64_t)IPX(ip, i*64+10) << 34;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*61+10) = (uint64_t)IPW(ip, i*64+10) >> 30 | (uint64_t)IPX(ip, i*64+11) << 31;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*61+11) = (uint64_t)IPW(ip, i*64+11) >> 33 | (uint64_t)IPX(ip, i*64+12) << 28;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*61+12) = (uint64_t)IPW(ip, i*64+12) >> 36 | (uint64_t)IPX(ip, i*64+13) << 25;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*61+13) = (uint64_t)IPW(ip, i*64+13) >> 39 | (uint64_t)IPX(ip, i*64+14) << 22;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*61+14) = (uint64_t)IPW(ip, i*64+14) >> 42 | (uint64_t)IPX(ip, i*64+15) << 19;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*61+15) = (uint64_t)IPW(ip, i*64+15) >> 45 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*61+16) = (uint64_t)IPW(ip, i*64+16) >> 48 | (uint64_t)IPX(ip, i*64+17) << 13;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*61+17) = (uint64_t)IPW(ip, i*64+17) >> 51 | (uint64_t)IPX(ip, i*64+18) << 10;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*61+18) = (uint64_t)IPW(ip, i*64+18) >> 54 | (uint64_t)IPX(ip, i*64+19) << 7;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*61+19) = (uint64_t)IPW(ip, i*64+19) >> 57 | (uint64_t)IPX(ip, i*64+20) << 4;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*61+20) = (uint64_t)IPW(ip, i*64+20) >> 60;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*61+20) |= (uint64_t)IPV(ip, i*64+21) << 1 | (uint64_t)IPX(ip, i*64+22) << 62;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*61+21) = (uint64_t)IPW(ip, i*64+22) >> 2 | (uint64_t)IPX(ip, i*64+23) << 59;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*61+22) = (uint64_t)IPW(ip, i*64+23) >> 5 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*61+23) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 53;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*61+24) = (uint64_t)IPW(ip, i*64+25) >> 11 | (uint64_t)IPX(ip, i*64+26) << 50;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*61+25) = (uint64_t)IPW(ip, i*64+26) >> 14 | (uint64_t)IPX(ip, i*64+27) << 47;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*61+26) = (uint64_t)IPW(ip, i*64+27) >> 17 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*61+27) = (uint64_t)IPW(ip, i*64+28) >> 20 | (uint64_t)IPX(ip, i*64+29) << 41;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*61+28) = (uint64_t)IPW(ip, i*64+29) >> 23 | (uint64_t)IPX(ip, i*64+30) << 38;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*61+29) = (uint64_t)IPW(ip, i*64+30) >> 26 | (uint64_t)IPX(ip, i*64+31) << 35;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*61+30) = (uint64_t)IPW(ip, i*64+31) >> 29;\
-}
-
-#define BITPACK64_61(ip, op, parm) { \
- BITBLK64_61(ip, 0, op, parm); IPI(ip); op += 61*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_62(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*31+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 62;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*31+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 2 | (uint64_t)IPX(ip, i*32+2) << 60;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*31+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 4 | (uint64_t)IPX(ip, i*32+3) << 58;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*31+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 6 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*31+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 8 | (uint64_t)IPX(ip, i*32+5) << 54;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*31+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 10 | (uint64_t)IPX(ip, i*32+6) << 52;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*31+ 6) = (uint64_t)IPW(ip, i*32+ 6) >> 12 | (uint64_t)IPX(ip, i*32+7) << 50;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*31+ 7) = (uint64_t)IPW(ip, i*32+ 7) >> 14 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*31+ 8) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 46;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*31+ 9) = (uint64_t)IPW(ip, i*32+ 9) >> 18 | (uint64_t)IPX(ip, i*32+10) << 44;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*31+10) = (uint64_t)IPW(ip, i*32+10) >> 20 | (uint64_t)IPX(ip, i*32+11) << 42;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*31+11) = (uint64_t)IPW(ip, i*32+11) >> 22 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*31+12) = (uint64_t)IPW(ip, i*32+12) >> 24 | (uint64_t)IPX(ip, i*32+13) << 38;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*31+13) = (uint64_t)IPW(ip, i*32+13) >> 26 | (uint64_t)IPX(ip, i*32+14) << 36;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*31+14) = (uint64_t)IPW(ip, i*32+14) >> 28 | (uint64_t)IPX(ip, i*32+15) << 34;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*31+15) = (uint64_t)IPW(ip, i*32+15) >> 30 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*31+16) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 30;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*31+17) = (uint64_t)IPW(ip, i*32+17) >> 34 | (uint64_t)IPX(ip, i*32+18) << 28;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*31+18) = (uint64_t)IPW(ip, i*32+18) >> 36 | (uint64_t)IPX(ip, i*32+19) << 26;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*31+19) = (uint64_t)IPW(ip, i*32+19) >> 38 | (uint64_t)IPX(ip, i*32+20) << 24;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*31+20) = (uint64_t)IPW(ip, i*32+20) >> 40 | (uint64_t)IPX(ip, i*32+21) << 22;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*31+21) = (uint64_t)IPW(ip, i*32+21) >> 42 | (uint64_t)IPX(ip, i*32+22) << 20;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*31+22) = (uint64_t)IPW(ip, i*32+22) >> 44 | (uint64_t)IPX(ip, i*32+23) << 18;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*31+23) = (uint64_t)IPW(ip, i*32+23) >> 46 | (uint64_t)IPX(ip, i*32+24) << 16;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*31+24) = (uint64_t)IPW(ip, i*32+24) >> 48 | (uint64_t)IPX(ip, i*32+25) << 14;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*31+25) = (uint64_t)IPW(ip, i*32+25) >> 50 | (uint64_t)IPX(ip, i*32+26) << 12;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*31+26) = (uint64_t)IPW(ip, i*32+26) >> 52 | (uint64_t)IPX(ip, i*32+27) << 10;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*31+27) = (uint64_t)IPW(ip, i*32+27) >> 54 | (uint64_t)IPX(ip, i*32+28) << 8;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*31+28) = (uint64_t)IPW(ip, i*32+28) >> 56 | (uint64_t)IPX(ip, i*32+29) << 6;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*31+29) = (uint64_t)IPW(ip, i*32+29) >> 58 | (uint64_t)IPX(ip, i*32+30) << 4;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*31+30) = (uint64_t)IPW(ip, i*32+30) >> 60;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*31+30) |= (uint64_t)IPV(ip, i*32+31) << 2;\
-}
-
-#define BITPACK64_62(ip, op, parm) { \
- BITBLK64_62(ip, 0, op, parm); IPI(ip); op += 62*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_63(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*63+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 63;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*63+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 1 | (uint64_t)IPX(ip, i*64+2) << 62;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*63+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 2 | (uint64_t)IPX(ip, i*64+3) << 61;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*63+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 3 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*63+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 4 | (uint64_t)IPX(ip, i*64+5) << 59;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*63+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 5 | (uint64_t)IPX(ip, i*64+6) << 58;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*63+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 6 | (uint64_t)IPX(ip, i*64+7) << 57;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*63+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 7 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*63+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 55;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*63+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 9 | (uint64_t)IPX(ip, i*64+10) << 54;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*63+10) = (uint64_t)IPW(ip, i*64+10) >> 10 | (uint64_t)IPX(ip, i*64+11) << 53;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*63+11) = (uint64_t)IPW(ip, i*64+11) >> 11 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*63+12) = (uint64_t)IPW(ip, i*64+12) >> 12 | (uint64_t)IPX(ip, i*64+13) << 51;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*63+13) = (uint64_t)IPW(ip, i*64+13) >> 13 | (uint64_t)IPX(ip, i*64+14) << 50;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*63+14) = (uint64_t)IPW(ip, i*64+14) >> 14 | (uint64_t)IPX(ip, i*64+15) << 49;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*63+15) = (uint64_t)IPW(ip, i*64+15) >> 15 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*63+16) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 47;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*63+17) = (uint64_t)IPW(ip, i*64+17) >> 17 | (uint64_t)IPX(ip, i*64+18) << 46;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*63+18) = (uint64_t)IPW(ip, i*64+18) >> 18 | (uint64_t)IPX(ip, i*64+19) << 45;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*63+19) = (uint64_t)IPW(ip, i*64+19) >> 19 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*63+20) = (uint64_t)IPW(ip, i*64+20) >> 20 | (uint64_t)IPX(ip, i*64+21) << 43;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*63+21) = (uint64_t)IPW(ip, i*64+21) >> 21 | (uint64_t)IPX(ip, i*64+22) << 42;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*63+22) = (uint64_t)IPW(ip, i*64+22) >> 22 | (uint64_t)IPX(ip, i*64+23) << 41;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*63+23) = (uint64_t)IPW(ip, i*64+23) >> 23 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*63+24) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 39;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*63+25) = (uint64_t)IPW(ip, i*64+25) >> 25 | (uint64_t)IPX(ip, i*64+26) << 38;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*63+26) = (uint64_t)IPW(ip, i*64+26) >> 26 | (uint64_t)IPX(ip, i*64+27) << 37;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*63+27) = (uint64_t)IPW(ip, i*64+27) >> 27 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*63+28) = (uint64_t)IPW(ip, i*64+28) >> 28 | (uint64_t)IPX(ip, i*64+29) << 35;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*63+29) = (uint64_t)IPW(ip, i*64+29) >> 29 | (uint64_t)IPX(ip, i*64+30) << 34;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*63+30) = (uint64_t)IPW(ip, i*64+30) >> 30 | (uint64_t)IPX(ip, i*64+31) << 33;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*63+31) = (uint64_t)IPW(ip, i*64+31) >> 31;\
-}
-
-#define BITPACK64_63(ip, op, parm) { \
- BITBLK64_63(ip, 0, op, parm); IPI(ip); op += 63*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_64(ip, i, op, parm) { ;\
- IP9(ip, i*1+ 0, parm); *((uint64_t *)op+i*1+ 0) = (uint64_t)IPV(ip, i*1+ 0) ;\
-}
-
-#define BITPACK64_64(ip, op, parm) { \
- BITBLK64_64(ip, 0, op, parm);\
- BITBLK64_64(ip, 1, op, parm);\
- BITBLK64_64(ip, 2, op, parm);\
- BITBLK64_64(ip, 3, op, parm);\
- BITBLK64_64(ip, 4, op, parm);\
- BITBLK64_64(ip, 5, op, parm);\
- BITBLK64_64(ip, 6, op, parm);\
- BITBLK64_64(ip, 7, op, parm);\
- BITBLK64_64(ip, 8, op, parm);\
- BITBLK64_64(ip, 9, op, parm);\
- BITBLK64_64(ip, 10, op, parm);\
- BITBLK64_64(ip, 11, op, parm);\
- BITBLK64_64(ip, 12, op, parm);\
- BITBLK64_64(ip, 13, op, parm);\
- BITBLK64_64(ip, 14, op, parm);\
- BITBLK64_64(ip, 15, op, parm);\
- BITBLK64_64(ip, 16, op, parm);\
- BITBLK64_64(ip, 17, op, parm);\
- BITBLK64_64(ip, 18, op, parm);\
- BITBLK64_64(ip, 19, op, parm);\
- BITBLK64_64(ip, 20, op, parm);\
- BITBLK64_64(ip, 21, op, parm);\
- BITBLK64_64(ip, 22, op, parm);\
- BITBLK64_64(ip, 23, op, parm);\
- BITBLK64_64(ip, 24, op, parm);\
- BITBLK64_64(ip, 25, op, parm);\
- BITBLK64_64(ip, 26, op, parm);\
- BITBLK64_64(ip, 27, op, parm);\
- BITBLK64_64(ip, 28, op, parm);\
- BITBLK64_64(ip, 29, op, parm);\
- BITBLK64_64(ip, 30, op, parm);\
- BITBLK64_64(ip, 31, op, parm); IPI(ip); op += 64*4/sizeof(op[0]);\
-}
-
-#ifndef DELTA
-#define USIZE 8
-unsigned char *TEMPLATE2(_BITPACK_,8_0)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,8_1)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint8_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_2)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint8_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_3)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint8_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_4)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint8_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_5)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint8_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_6)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint8_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_7)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint8_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_8)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint8_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F8 TEMPLATE2(_BITPACK_,a8)[] = {
- &TEMPLATE2(_BITPACK_,8_0),
- &TEMPLATE2(_BITPACK_,8_1),
- &TEMPLATE2(_BITPACK_,8_2),
- &TEMPLATE2(_BITPACK_,8_3),
- &TEMPLATE2(_BITPACK_,8_4),
- &TEMPLATE2(_BITPACK_,8_5),
- &TEMPLATE2(_BITPACK_,8_6),
- &TEMPLATE2(_BITPACK_,8_7),
- &TEMPLATE2(_BITPACK_,8_8)
-};
-unsigned char *TEMPLATE2(_BITPACK_,8)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a8)[ b](in, n, out); }
-
-#define USIZE 16
-unsigned char *TEMPLATE2(_BITPACK_,16_0)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,16_1)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint16_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_2)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint16_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_3)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint16_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_4)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint16_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_5)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint16_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_6)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint16_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_7)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint16_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_8)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint16_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_9)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint16_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_10)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint16_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_11)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint16_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_12)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint16_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_13)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint16_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_14)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint16_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_15)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint16_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_16)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint16_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F16 TEMPLATE2(_BITPACK_,a16)[] = {
- &TEMPLATE2(_BITPACK_,16_0),
- &TEMPLATE2(_BITPACK_,16_1),
- &TEMPLATE2(_BITPACK_,16_2),
- &TEMPLATE2(_BITPACK_,16_3),
- &TEMPLATE2(_BITPACK_,16_4),
- &TEMPLATE2(_BITPACK_,16_5),
- &TEMPLATE2(_BITPACK_,16_6),
- &TEMPLATE2(_BITPACK_,16_7),
- &TEMPLATE2(_BITPACK_,16_8),
- &TEMPLATE2(_BITPACK_,16_9),
- &TEMPLATE2(_BITPACK_,16_10),
- &TEMPLATE2(_BITPACK_,16_11),
- &TEMPLATE2(_BITPACK_,16_12),
- &TEMPLATE2(_BITPACK_,16_13),
- &TEMPLATE2(_BITPACK_,16_14),
- &TEMPLATE2(_BITPACK_,16_15),
- &TEMPLATE2(_BITPACK_,16_16)
-};
-unsigned char *TEMPLATE2(_BITPACK_,16)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a16)[ b](in, n, out); }
-
-#define USIZE 32
-unsigned char *TEMPLATE2(_BITPACK_,32_0)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,32_1)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint32_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_2)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint32_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_3)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint32_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_4)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint32_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_5)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint32_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_6)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint32_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_7)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint32_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_8)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint32_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_9)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint32_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_10)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint32_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_11)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint32_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_12)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint32_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_13)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint32_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_14)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint32_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_15)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint32_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_16)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint32_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_17)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*17); uint32_t v,x;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_18)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*18); uint32_t v,x;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_19)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*19); uint32_t v,x;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_20)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*20); uint32_t v,x;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_21)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*21); uint32_t v,x;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_22)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*22); uint32_t v,x;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_23)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*23); uint32_t v,x;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_24)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*24); uint32_t v,x;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_25)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*25); uint32_t v,x;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_26)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*26); uint32_t v,x;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_27)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*27); uint32_t v,x;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_28)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*28); uint32_t v,x;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_29)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*29); uint32_t v,x;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_30)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*30); uint32_t v,x;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_31)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*31); uint32_t v,x;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_32)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*32); uint32_t v,x;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F32 TEMPLATE2(_BITPACK_,a32)[] = {
- &TEMPLATE2(_BITPACK_,32_0),
- &TEMPLATE2(_BITPACK_,32_1),
- &TEMPLATE2(_BITPACK_,32_2),
- &TEMPLATE2(_BITPACK_,32_3),
- &TEMPLATE2(_BITPACK_,32_4),
- &TEMPLATE2(_BITPACK_,32_5),
- &TEMPLATE2(_BITPACK_,32_6),
- &TEMPLATE2(_BITPACK_,32_7),
- &TEMPLATE2(_BITPACK_,32_8),
- &TEMPLATE2(_BITPACK_,32_9),
- &TEMPLATE2(_BITPACK_,32_10),
- &TEMPLATE2(_BITPACK_,32_11),
- &TEMPLATE2(_BITPACK_,32_12),
- &TEMPLATE2(_BITPACK_,32_13),
- &TEMPLATE2(_BITPACK_,32_14),
- &TEMPLATE2(_BITPACK_,32_15),
- &TEMPLATE2(_BITPACK_,32_16),
- &TEMPLATE2(_BITPACK_,32_17),
- &TEMPLATE2(_BITPACK_,32_18),
- &TEMPLATE2(_BITPACK_,32_19),
- &TEMPLATE2(_BITPACK_,32_20),
- &TEMPLATE2(_BITPACK_,32_21),
- &TEMPLATE2(_BITPACK_,32_22),
- &TEMPLATE2(_BITPACK_,32_23),
- &TEMPLATE2(_BITPACK_,32_24),
- &TEMPLATE2(_BITPACK_,32_25),
- &TEMPLATE2(_BITPACK_,32_26),
- &TEMPLATE2(_BITPACK_,32_27),
- &TEMPLATE2(_BITPACK_,32_28),
- &TEMPLATE2(_BITPACK_,32_29),
- &TEMPLATE2(_BITPACK_,32_30),
- &TEMPLATE2(_BITPACK_,32_31),
- &TEMPLATE2(_BITPACK_,32_32)
-};
-unsigned char *TEMPLATE2(_BITPACK_,32)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a32)[ b](in, n, out); }
-
-#define USIZE 64
-unsigned char *TEMPLATE2(_BITPACK_,64_0)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,64_1)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint64_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_2)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint64_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_3)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint64_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_4)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint64_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_5)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint64_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_6)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint64_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_7)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint64_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_8)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint64_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_9)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint64_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_10)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint64_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_11)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint64_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_12)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint64_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_13)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint64_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_14)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint64_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_15)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint64_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_16)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint64_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_17)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*17); uint64_t v,x;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_18)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*18); uint64_t v,x;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_19)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*19); uint64_t v,x;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_20)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*20); uint64_t v,x;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_21)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*21); uint64_t v,x;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_22)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*22); uint64_t v,x;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_23)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*23); uint64_t v,x;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_24)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*24); uint64_t v,x;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_25)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*25); uint64_t v,x;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_26)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*26); uint64_t v,x;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_27)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*27); uint64_t v,x;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_28)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*28); uint64_t v,x;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_29)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*29); uint64_t v,x;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_30)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*30); uint64_t v,x;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_31)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*31); uint64_t v,x;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_32)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*32); uint64_t v,x;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_33)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*33); uint64_t v,x;do { BITPACK64_33( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_34)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*34); uint64_t v,x;do { BITPACK64_34( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_35)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*35); uint64_t v,x;do { BITPACK64_35( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_36)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*36); uint64_t v,x;do { BITPACK64_36( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_37)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*37); uint64_t v,x;do { BITPACK64_37( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_38)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*38); uint64_t v,x;do { BITPACK64_38( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_39)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*39); uint64_t v,x;do { BITPACK64_39( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_40)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*40); uint64_t v,x;do { BITPACK64_40( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_41)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*41); uint64_t v,x;do { BITPACK64_41( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_42)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*42); uint64_t v,x;do { BITPACK64_42( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_43)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*43); uint64_t v,x;do { BITPACK64_43( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_44)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*44); uint64_t v,x;do { BITPACK64_44( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_45)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*45); uint64_t v,x;do { BITPACK64_45( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_46)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*46); uint64_t v,x;do { BITPACK64_46( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_47)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*47); uint64_t v,x;do { BITPACK64_47( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_48)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*48); uint64_t v,x;do { BITPACK64_48( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_49)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*49); uint64_t v,x;do { BITPACK64_49( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_50)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*50); uint64_t v,x;do { BITPACK64_50( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_51)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*51); uint64_t v,x;do { BITPACK64_51( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_52)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*52); uint64_t v,x;do { BITPACK64_52( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_53)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*53); uint64_t v,x;do { BITPACK64_53( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_54)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*54); uint64_t v,x;do { BITPACK64_54( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_55)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*55); uint64_t v,x;do { BITPACK64_55( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_56)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*56); uint64_t v,x;do { BITPACK64_56( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_57)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*57); uint64_t v,x;do { BITPACK64_57( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_58)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*58); uint64_t v,x;do { BITPACK64_58( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_59)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*59); uint64_t v,x;do { BITPACK64_59( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_60)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*60); uint64_t v,x;do { BITPACK64_60( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_61)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*61); uint64_t v,x;do { BITPACK64_61( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_62)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*62); uint64_t v,x;do { BITPACK64_62( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_63)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*63); uint64_t v,x;do { BITPACK64_63( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_64)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*64); uint64_t v,x;do { BITPACK64_64( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F64 TEMPLATE2(_BITPACK_,a64)[] = {
- &TEMPLATE2(_BITPACK_,64_0),
- &TEMPLATE2(_BITPACK_,64_1),
- &TEMPLATE2(_BITPACK_,64_2),
- &TEMPLATE2(_BITPACK_,64_3),
- &TEMPLATE2(_BITPACK_,64_4),
- &TEMPLATE2(_BITPACK_,64_5),
- &TEMPLATE2(_BITPACK_,64_6),
- &TEMPLATE2(_BITPACK_,64_7),
- &TEMPLATE2(_BITPACK_,64_8),
- &TEMPLATE2(_BITPACK_,64_9),
- &TEMPLATE2(_BITPACK_,64_10),
- &TEMPLATE2(_BITPACK_,64_11),
- &TEMPLATE2(_BITPACK_,64_12),
- &TEMPLATE2(_BITPACK_,64_13),
- &TEMPLATE2(_BITPACK_,64_14),
- &TEMPLATE2(_BITPACK_,64_15),
- &TEMPLATE2(_BITPACK_,64_16),
- &TEMPLATE2(_BITPACK_,64_17),
- &TEMPLATE2(_BITPACK_,64_18),
- &TEMPLATE2(_BITPACK_,64_19),
- &TEMPLATE2(_BITPACK_,64_20),
- &TEMPLATE2(_BITPACK_,64_21),
- &TEMPLATE2(_BITPACK_,64_22),
- &TEMPLATE2(_BITPACK_,64_23),
- &TEMPLATE2(_BITPACK_,64_24),
- &TEMPLATE2(_BITPACK_,64_25),
- &TEMPLATE2(_BITPACK_,64_26),
- &TEMPLATE2(_BITPACK_,64_27),
- &TEMPLATE2(_BITPACK_,64_28),
- &TEMPLATE2(_BITPACK_,64_29),
- &TEMPLATE2(_BITPACK_,64_30),
- &TEMPLATE2(_BITPACK_,64_31),
- &TEMPLATE2(_BITPACK_,64_32),
- &TEMPLATE2(_BITPACK_,64_33),
- &TEMPLATE2(_BITPACK_,64_34),
- &TEMPLATE2(_BITPACK_,64_35),
- &TEMPLATE2(_BITPACK_,64_36),
- &TEMPLATE2(_BITPACK_,64_37),
- &TEMPLATE2(_BITPACK_,64_38),
- &TEMPLATE2(_BITPACK_,64_39),
- &TEMPLATE2(_BITPACK_,64_40),
- &TEMPLATE2(_BITPACK_,64_41),
- &TEMPLATE2(_BITPACK_,64_42),
- &TEMPLATE2(_BITPACK_,64_43),
- &TEMPLATE2(_BITPACK_,64_44),
- &TEMPLATE2(_BITPACK_,64_45),
- &TEMPLATE2(_BITPACK_,64_46),
- &TEMPLATE2(_BITPACK_,64_47),
- &TEMPLATE2(_BITPACK_,64_48),
- &TEMPLATE2(_BITPACK_,64_49),
- &TEMPLATE2(_BITPACK_,64_50),
- &TEMPLATE2(_BITPACK_,64_51),
- &TEMPLATE2(_BITPACK_,64_52),
- &TEMPLATE2(_BITPACK_,64_53),
- &TEMPLATE2(_BITPACK_,64_54),
- &TEMPLATE2(_BITPACK_,64_55),
- &TEMPLATE2(_BITPACK_,64_56),
- &TEMPLATE2(_BITPACK_,64_57),
- &TEMPLATE2(_BITPACK_,64_58),
- &TEMPLATE2(_BITPACK_,64_59),
- &TEMPLATE2(_BITPACK_,64_60),
- &TEMPLATE2(_BITPACK_,64_61),
- &TEMPLATE2(_BITPACK_,64_62),
- &TEMPLATE2(_BITPACK_,64_63),
- &TEMPLATE2(_BITPACK_,64_64)
-};
-unsigned char *TEMPLATE2(_BITPACK_,64)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a64)[ b](in, n, out); }
-
-#else
-#define USIZE 8
-unsigned char *TEMPLATE2(_BITPACK_,8_0)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,8_1)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*1); uint8_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_2)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*2); uint8_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_3)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*3); uint8_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_4)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*4); uint8_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_5)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*5); uint8_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_6)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*6); uint8_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_7)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*7); uint8_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_8)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*8); uint8_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D8 TEMPLATE2(_BITPACK_,a8)[] = {
- &TEMPLATE2(_BITPACK_,8_0),
- &TEMPLATE2(_BITPACK_,8_1),
- &TEMPLATE2(_BITPACK_,8_2),
- &TEMPLATE2(_BITPACK_,8_3),
- &TEMPLATE2(_BITPACK_,8_4),
- &TEMPLATE2(_BITPACK_,8_5),
- &TEMPLATE2(_BITPACK_,8_6),
- &TEMPLATE2(_BITPACK_,8_7),
- &TEMPLATE2(_BITPACK_,8_8)
-};
-unsigned char *TEMPLATE2(_BITPACK_,8)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a8)[ b](in, n, out, start); }
-
-#define USIZE 16
-unsigned char *TEMPLATE2(_BITPACK_,16_0)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,16_1)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*1); uint16_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_2)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*2); uint16_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_3)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*3); uint16_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_4)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*4); uint16_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_5)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*5); uint16_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_6)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*6); uint16_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_7)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*7); uint16_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_8)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*8); uint16_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_9)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*9); uint16_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_10)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*10); uint16_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_11)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*11); uint16_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_12)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*12); uint16_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_13)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*13); uint16_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_14)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*14); uint16_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_15)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*15); uint16_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_16)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*16); uint16_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D16 TEMPLATE2(_BITPACK_,a16)[] = {
- &TEMPLATE2(_BITPACK_,16_0),
- &TEMPLATE2(_BITPACK_,16_1),
- &TEMPLATE2(_BITPACK_,16_2),
- &TEMPLATE2(_BITPACK_,16_3),
- &TEMPLATE2(_BITPACK_,16_4),
- &TEMPLATE2(_BITPACK_,16_5),
- &TEMPLATE2(_BITPACK_,16_6),
- &TEMPLATE2(_BITPACK_,16_7),
- &TEMPLATE2(_BITPACK_,16_8),
- &TEMPLATE2(_BITPACK_,16_9),
- &TEMPLATE2(_BITPACK_,16_10),
- &TEMPLATE2(_BITPACK_,16_11),
- &TEMPLATE2(_BITPACK_,16_12),
- &TEMPLATE2(_BITPACK_,16_13),
- &TEMPLATE2(_BITPACK_,16_14),
- &TEMPLATE2(_BITPACK_,16_15),
- &TEMPLATE2(_BITPACK_,16_16)
-};
-unsigned char *TEMPLATE2(_BITPACK_,16)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a16)[ b](in, n, out, start); }
-
-#define USIZE 32
-unsigned char *TEMPLATE2(_BITPACK_,32_0)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,32_1)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*1); uint32_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_2)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*2); uint32_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_3)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*3); uint32_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_4)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*4); uint32_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_5)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*5); uint32_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_6)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*6); uint32_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_7)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*7); uint32_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_8)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*8); uint32_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_9)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*9); uint32_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_10)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*10); uint32_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_11)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*11); uint32_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_12)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*12); uint32_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_13)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*13); uint32_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_14)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*14); uint32_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_15)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*15); uint32_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_16)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*16); uint32_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_17)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*17); uint32_t v,x=0;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_18)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*18); uint32_t v,x=0;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_19)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*19); uint32_t v,x=0;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_20)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*20); uint32_t v,x=0;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_21)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*21); uint32_t v,x=0;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_22)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*22); uint32_t v,x=0;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_23)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*23); uint32_t v,x=0;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_24)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*24); uint32_t v,x=0;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_25)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*25); uint32_t v,x=0;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_26)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*26); uint32_t v,x=0;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_27)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*27); uint32_t v,x=0;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_28)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*28); uint32_t v,x=0;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_29)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*29); uint32_t v,x=0;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_30)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*30); uint32_t v,x=0;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_31)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*31); uint32_t v,x=0;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_32)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*32); uint32_t v,x=0;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D32 TEMPLATE2(_BITPACK_,a32)[] = {
- &TEMPLATE2(_BITPACK_,32_0),
- &TEMPLATE2(_BITPACK_,32_1),
- &TEMPLATE2(_BITPACK_,32_2),
- &TEMPLATE2(_BITPACK_,32_3),
- &TEMPLATE2(_BITPACK_,32_4),
- &TEMPLATE2(_BITPACK_,32_5),
- &TEMPLATE2(_BITPACK_,32_6),
- &TEMPLATE2(_BITPACK_,32_7),
- &TEMPLATE2(_BITPACK_,32_8),
- &TEMPLATE2(_BITPACK_,32_9),
- &TEMPLATE2(_BITPACK_,32_10),
- &TEMPLATE2(_BITPACK_,32_11),
- &TEMPLATE2(_BITPACK_,32_12),
- &TEMPLATE2(_BITPACK_,32_13),
- &TEMPLATE2(_BITPACK_,32_14),
- &TEMPLATE2(_BITPACK_,32_15),
- &TEMPLATE2(_BITPACK_,32_16),
- &TEMPLATE2(_BITPACK_,32_17),
- &TEMPLATE2(_BITPACK_,32_18),
- &TEMPLATE2(_BITPACK_,32_19),
- &TEMPLATE2(_BITPACK_,32_20),
- &TEMPLATE2(_BITPACK_,32_21),
- &TEMPLATE2(_BITPACK_,32_22),
- &TEMPLATE2(_BITPACK_,32_23),
- &TEMPLATE2(_BITPACK_,32_24),
- &TEMPLATE2(_BITPACK_,32_25),
- &TEMPLATE2(_BITPACK_,32_26),
- &TEMPLATE2(_BITPACK_,32_27),
- &TEMPLATE2(_BITPACK_,32_28),
- &TEMPLATE2(_BITPACK_,32_29),
- &TEMPLATE2(_BITPACK_,32_30),
- &TEMPLATE2(_BITPACK_,32_31),
- &TEMPLATE2(_BITPACK_,32_32)
-};
-unsigned char *TEMPLATE2(_BITPACK_,32)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a32)[ b](in, n, out, start); }
-
-#define USIZE 64
-unsigned char *TEMPLATE2(_BITPACK_,64_0)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,64_1)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*1); uint64_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_2)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*2); uint64_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_3)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*3); uint64_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_4)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*4); uint64_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_5)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*5); uint64_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_6)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*6); uint64_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_7)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*7); uint64_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_8)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*8); uint64_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_9)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*9); uint64_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_10)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*10); uint64_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_11)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*11); uint64_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_12)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*12); uint64_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_13)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*13); uint64_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_14)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*14); uint64_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_15)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*15); uint64_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_16)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*16); uint64_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_17)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*17); uint64_t v,x=0;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_18)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*18); uint64_t v,x=0;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_19)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*19); uint64_t v,x=0;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_20)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*20); uint64_t v,x=0;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_21)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*21); uint64_t v,x=0;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_22)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*22); uint64_t v,x=0;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_23)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*23); uint64_t v,x=0;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_24)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*24); uint64_t v,x=0;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_25)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*25); uint64_t v,x=0;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_26)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*26); uint64_t v,x=0;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_27)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*27); uint64_t v,x=0;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_28)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*28); uint64_t v,x=0;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_29)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*29); uint64_t v,x=0;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_30)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*30); uint64_t v,x=0;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_31)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*31); uint64_t v,x=0;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_32)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*32); uint64_t v,x=0;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_33)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*33); uint64_t v,x=0;do { BITPACK64_33( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_34)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*34); uint64_t v,x=0;do { BITPACK64_34( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_35)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*35); uint64_t v,x=0;do { BITPACK64_35( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_36)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*36); uint64_t v,x=0;do { BITPACK64_36( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_37)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*37); uint64_t v,x=0;do { BITPACK64_37( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_38)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*38); uint64_t v,x=0;do { BITPACK64_38( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_39)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*39); uint64_t v,x=0;do { BITPACK64_39( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_40)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*40); uint64_t v,x=0;do { BITPACK64_40( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_41)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*41); uint64_t v,x=0;do { BITPACK64_41( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_42)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*42); uint64_t v,x=0;do { BITPACK64_42( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_43)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*43); uint64_t v,x=0;do { BITPACK64_43( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_44)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*44); uint64_t v,x=0;do { BITPACK64_44( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_45)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*45); uint64_t v,x=0;do { BITPACK64_45( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_46)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*46); uint64_t v,x=0;do { BITPACK64_46( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_47)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*47); uint64_t v,x=0;do { BITPACK64_47( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_48)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*48); uint64_t v,x=0;do { BITPACK64_48( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_49)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*49); uint64_t v,x=0;do { BITPACK64_49( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_50)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*50); uint64_t v,x=0;do { BITPACK64_50( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_51)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*51); uint64_t v,x=0;do { BITPACK64_51( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_52)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*52); uint64_t v,x=0;do { BITPACK64_52( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_53)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*53); uint64_t v,x=0;do { BITPACK64_53( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_54)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*54); uint64_t v,x=0;do { BITPACK64_54( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_55)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*55); uint64_t v,x=0;do { BITPACK64_55( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_56)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*56); uint64_t v,x=0;do { BITPACK64_56( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_57)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*57); uint64_t v,x=0;do { BITPACK64_57( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_58)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*58); uint64_t v,x=0;do { BITPACK64_58( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_59)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*59); uint64_t v,x=0;do { BITPACK64_59( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_60)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*60); uint64_t v,x=0;do { BITPACK64_60( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_61)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*61); uint64_t v,x=0;do { BITPACK64_61( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_62)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*62); uint64_t v,x=0;do { BITPACK64_62( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_63)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*63); uint64_t v,x=0;do { BITPACK64_63( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_64)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*64); uint64_t v,x=0;do { BITPACK64_64( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D64 TEMPLATE2(_BITPACK_,a64)[] = {
- &TEMPLATE2(_BITPACK_,64_0),
- &TEMPLATE2(_BITPACK_,64_1),
- &TEMPLATE2(_BITPACK_,64_2),
- &TEMPLATE2(_BITPACK_,64_3),
- &TEMPLATE2(_BITPACK_,64_4),
- &TEMPLATE2(_BITPACK_,64_5),
- &TEMPLATE2(_BITPACK_,64_6),
- &TEMPLATE2(_BITPACK_,64_7),
- &TEMPLATE2(_BITPACK_,64_8),
- &TEMPLATE2(_BITPACK_,64_9),
- &TEMPLATE2(_BITPACK_,64_10),
- &TEMPLATE2(_BITPACK_,64_11),
- &TEMPLATE2(_BITPACK_,64_12),
- &TEMPLATE2(_BITPACK_,64_13),
- &TEMPLATE2(_BITPACK_,64_14),
- &TEMPLATE2(_BITPACK_,64_15),
- &TEMPLATE2(_BITPACK_,64_16),
- &TEMPLATE2(_BITPACK_,64_17),
- &TEMPLATE2(_BITPACK_,64_18),
- &TEMPLATE2(_BITPACK_,64_19),
- &TEMPLATE2(_BITPACK_,64_20),
- &TEMPLATE2(_BITPACK_,64_21),
- &TEMPLATE2(_BITPACK_,64_22),
- &TEMPLATE2(_BITPACK_,64_23),
- &TEMPLATE2(_BITPACK_,64_24),
- &TEMPLATE2(_BITPACK_,64_25),
- &TEMPLATE2(_BITPACK_,64_26),
- &TEMPLATE2(_BITPACK_,64_27),
- &TEMPLATE2(_BITPACK_,64_28),
- &TEMPLATE2(_BITPACK_,64_29),
- &TEMPLATE2(_BITPACK_,64_30),
- &TEMPLATE2(_BITPACK_,64_31),
- &TEMPLATE2(_BITPACK_,64_32),
- &TEMPLATE2(_BITPACK_,64_33),
- &TEMPLATE2(_BITPACK_,64_34),
- &TEMPLATE2(_BITPACK_,64_35),
- &TEMPLATE2(_BITPACK_,64_36),
- &TEMPLATE2(_BITPACK_,64_37),
- &TEMPLATE2(_BITPACK_,64_38),
- &TEMPLATE2(_BITPACK_,64_39),
- &TEMPLATE2(_BITPACK_,64_40),
- &TEMPLATE2(_BITPACK_,64_41),
- &TEMPLATE2(_BITPACK_,64_42),
- &TEMPLATE2(_BITPACK_,64_43),
- &TEMPLATE2(_BITPACK_,64_44),
- &TEMPLATE2(_BITPACK_,64_45),
- &TEMPLATE2(_BITPACK_,64_46),
- &TEMPLATE2(_BITPACK_,64_47),
- &TEMPLATE2(_BITPACK_,64_48),
- &TEMPLATE2(_BITPACK_,64_49),
- &TEMPLATE2(_BITPACK_,64_50),
- &TEMPLATE2(_BITPACK_,64_51),
- &TEMPLATE2(_BITPACK_,64_52),
- &TEMPLATE2(_BITPACK_,64_53),
- &TEMPLATE2(_BITPACK_,64_54),
- &TEMPLATE2(_BITPACK_,64_55),
- &TEMPLATE2(_BITPACK_,64_56),
- &TEMPLATE2(_BITPACK_,64_57),
- &TEMPLATE2(_BITPACK_,64_58),
- &TEMPLATE2(_BITPACK_,64_59),
- &TEMPLATE2(_BITPACK_,64_60),
- &TEMPLATE2(_BITPACK_,64_61),
- &TEMPLATE2(_BITPACK_,64_62),
- &TEMPLATE2(_BITPACK_,64_63),
- &TEMPLATE2(_BITPACK_,64_64)
-};
-unsigned char *TEMPLATE2(_BITPACK_,64)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a64)[ b](in, n, out, start); }
-
-#endif
-#endif //IP9
-
-//---------------------------------------------------------
-#define BITBLK128V16_1(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 1));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 2));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 3));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 5));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 6));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 7));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 9));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 10));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 11));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 12));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 13));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 14));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 15)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_1(ip, op, parm) {\
- BITBLK128V16_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_2(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 1, iv), 2));\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 2, iv), 4));\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 3, iv), 6));\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 4, iv), 8));\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 5, iv), 10));\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 6, iv), 12));\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 14)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_2(ip, op, parm) {\
- BITBLK128V16_2(ip, 0, op, parm);\
- BITBLK128V16_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_3(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 3));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 6));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 9));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 12));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 2));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 5));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 11));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 1));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 7));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 10));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 13)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_3(ip, op, parm) {\
- BITBLK128V16_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_4(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*4+ 0, iv, parm); ov = IP16(ip, i*4+ 0, iv);\
- VI16(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 1, iv), 4));\
- VI16(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 2, iv), 8));\
- VI16(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 3, iv), 12)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_4(ip, op, parm) {\
- BITBLK128V16_4(ip, 0, op, parm);\
- BITBLK128V16_4(ip, 1, op, parm);\
- BITBLK128V16_4(ip, 2, op, parm);\
- BITBLK128V16_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_5(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 5));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 10));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 9));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 3));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 2));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 7));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 1));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 6));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 11)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_5(ip, op, parm) {\
- BITBLK128V16_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_6(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 1, iv), 6));\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 3, iv), 2));\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 4, iv), 8));\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 5, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 6, iv), 4));\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 10)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_6(ip, op, parm) {\
- BITBLK128V16_6(ip, 0, op, parm);\
- BITBLK128V16_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_7(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 7));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 5));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 3));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 1));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 6));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 2));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 9)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_7(ip, op, parm) {\
- BITBLK128V16_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_8(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*2+ 0, iv, parm); ov = IP16(ip, i*2+ 0, iv);\
- VI16(ip, i*2+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*2+ 1, iv), 8)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_8(ip, op, parm) {\
- BITBLK128V16_8(ip, 0, op, parm);\
- BITBLK128V16_8(ip, 1, op, parm);\
- BITBLK128V16_8(ip, 2, op, parm);\
- BITBLK128V16_8(ip, 3, op, parm);\
- BITBLK128V16_8(ip, 4, op, parm);\
- BITBLK128V16_8(ip, 5, op, parm);\
- BITBLK128V16_8(ip, 6, op, parm);\
- BITBLK128V16_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_9(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 2));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 6));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 1));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 3));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 5));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 7)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_9(ip, op, parm) {\
- BITBLK128V16_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_10(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 1, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 2, iv), 4));\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 3, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 5, iv), 2));\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 6, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 6)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_10(ip, op, parm) {\
- BITBLK128V16_10(ip, 0, op, parm);\
- BITBLK128V16_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_11(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 1));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 2));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 3));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 5)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_11(ip, op, parm) {\
- BITBLK128V16_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_12(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*4+ 0, iv, parm); ov = IP16(ip, i*4+ 0, iv);\
- VI16(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*4+ 1, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*4+ 2, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 3, iv), 4)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_12(ip, op, parm) {\
- BITBLK128V16_12(ip, 0, op, parm);\
- BITBLK128V16_12(ip, 1, op, parm);\
- BITBLK128V16_12(ip, 2, op, parm);\
- BITBLK128V16_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_13(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 1));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 11);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 2));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 3)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_13(ip, op, parm) {\
- BITBLK128V16_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_14(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 1, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 3, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 5, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 6, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 2)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_14(ip, op, parm) {\
- BITBLK128V16_14(ip, 0, op, parm);\
- BITBLK128V16_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_15(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 11);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 3)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 13);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 2)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 14);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 1)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_15(ip, op, parm) {\
- BITBLK128V16_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_16(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*1+ 0, iv, parm); ov = IP16(ip, i*1+ 0, iv); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_16(ip, op, parm) {\
- BITBLK128V16_16(ip, 0, op, parm);\
- BITBLK128V16_16(ip, 1, op, parm);\
- BITBLK128V16_16(ip, 2, op, parm);\
- BITBLK128V16_16(ip, 3, op, parm);\
- BITBLK128V16_16(ip, 4, op, parm);\
- BITBLK128V16_16(ip, 5, op, parm);\
- BITBLK128V16_16(ip, 6, op, parm);\
- BITBLK128V16_16(ip, 7, op, parm);\
- BITBLK128V16_16(ip, 8, op, parm);\
- BITBLK128V16_16(ip, 9, op, parm);\
- BITBLK128V16_16(ip, 10, op, parm);\
- BITBLK128V16_16(ip, 11, op, parm);\
- BITBLK128V16_16(ip, 12, op, parm);\
- BITBLK128V16_16(ip, 13, op, parm);\
- BITBLK128V16_16(ip, 14, op, parm);\
- BITBLK128V16_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_1(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 1));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 3));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 5));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 7));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 9));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 11));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 13));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 15));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 17));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 18));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 19));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 21));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 22));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 23));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 25));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 26));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 27));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 28));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 29));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 30));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 31)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_1(ip, op, parm) {\
- BITBLK128V32_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_2(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 2));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 6));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 10));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 14));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 18));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 20));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 22));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 24));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 26));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 28));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 30)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_2(ip, op, parm) {\
- BITBLK128V32_2(ip, 0, op, parm);\
- BITBLK128V32_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_3(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 3));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 9));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 15));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 18));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 21));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 27));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 1));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 7));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 13));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 19));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 22));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 25));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 28));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 5));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 11));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 14));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 17));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 23));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 26));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 29)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_3(ip, op, parm) {\
- BITBLK128V32_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_4(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 1, iv), 4));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 3, iv), 12));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 5, iv), 20));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 6, iv), 24));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 28)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_4(ip, op, parm) {\
- BITBLK128V32_4(ip, 0, op, parm);\
- BITBLK128V32_4(ip, 1, op, parm);\
- BITBLK128V32_4(ip, 2, op, parm);\
- BITBLK128V32_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_5(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 5));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 15));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 20));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 25));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 3));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 13));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 18));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 23));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 1));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 11));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 21));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 26));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 9));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 14));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 19));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 7));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 17));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 22));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 27)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_5(ip, op, parm) {\
- BITBLK128V32_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_6(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 6));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 12));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 18));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 24));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 10));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 22));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 2));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 14));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 20));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 26)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_6(ip, op, parm) {\
- BITBLK128V32_6(ip, 0, op, parm);\
- BITBLK128V32_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_7(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 7));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 14));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 21));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 3));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 10));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 17));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 13));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 20));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 9));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 23));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 5));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 19));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 1));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 15));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 22));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 11));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 18));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 25)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_7(ip, op, parm) {\
- BITBLK128V32_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_8(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 1, iv), 8));\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 2, iv), 16));\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 3, iv), 24)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_8(ip, op, parm) {\
- BITBLK128V32_8(ip, 0, op, parm);\
- BITBLK128V32_8(ip, 1, op, parm);\
- BITBLK128V32_8(ip, 2, op, parm);\
- BITBLK128V32_8(ip, 3, op, parm);\
- BITBLK128V32_8(ip, 4, op, parm);\
- BITBLK128V32_8(ip, 5, op, parm);\
- BITBLK128V32_8(ip, 6, op, parm);\
- BITBLK128V32_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_9(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 9));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 18));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 13));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 22));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 17));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 3));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 21));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 7));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 11));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 15));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 1));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 10));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 19));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 5));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 14));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 23)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_9(ip, op, parm) {\
- BITBLK128V32_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_10(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 10));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 20));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 18));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 6));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 14));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 2));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 12));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 22)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_10(ip, op, parm) {\
- BITBLK128V32_10(ip, 0, op, parm);\
- BITBLK128V32_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_11(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 11));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 1));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 13));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 3));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 14));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 15));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 5));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 6));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 17));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 7));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 18));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 19));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 9));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 10));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 21)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_11(ip, op, parm) {\
- BITBLK128V32_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_12(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 1, iv), 12));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 3, iv), 4));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 6, iv), 8));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 20)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_12(ip, op, parm) {\
- BITBLK128V32_12(ip, 0, op, parm);\
- BITBLK128V32_12(ip, 1, op, parm);\
- BITBLK128V32_12(ip, 2, op, parm);\
- BITBLK128V32_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_13(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 13));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 7));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 1));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 14));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 15));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 9));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 3));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 10));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 17));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 11));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 5));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 18));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 6));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 19)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_13(ip, op, parm) {\
- BITBLK128V32_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_14(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 14));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 10));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 6));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 2));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 12));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 4));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 18)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_14(ip, op, parm) {\
- BITBLK128V32_14(ip, 0, op, parm);\
- BITBLK128V32_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_15(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 15));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 13));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 11));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 9));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 7));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 5));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 3));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 1));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 14));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 10));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 6));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 2));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 17)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_15(ip, op, parm) {\
- BITBLK128V32_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_16(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*2+ 0, iv, parm); ov = IP32(ip, i*2+ 0, iv);\
- VI32(ip, i*2+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*2+ 1, iv), 16)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_16(ip, op, parm) {\
- BITBLK128V32_16(ip, 0, op, parm);\
- BITBLK128V32_16(ip, 1, op, parm);\
- BITBLK128V32_16(ip, 2, op, parm);\
- BITBLK128V32_16(ip, 3, op, parm);\
- BITBLK128V32_16(ip, 4, op, parm);\
- BITBLK128V32_16(ip, 5, op, parm);\
- BITBLK128V32_16(ip, 6, op, parm);\
- BITBLK128V32_16(ip, 7, op, parm);\
- BITBLK128V32_16(ip, 8, op, parm);\
- BITBLK128V32_16(ip, 9, op, parm);\
- BITBLK128V32_16(ip, 10, op, parm);\
- BITBLK128V32_16(ip, 11, op, parm);\
- BITBLK128V32_16(ip, 12, op, parm);\
- BITBLK128V32_16(ip, 13, op, parm);\
- BITBLK128V32_16(ip, 14, op, parm);\
- BITBLK128V32_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_17(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 1));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 3));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 5));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 7));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 9));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 11));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 13));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 15)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_17(ip, op, parm) {\
- BITBLK128V32_17(ip, 0, op, parm); IPPE(ip); OPPE(op += 17*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_18(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 2));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 6));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 10));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 14)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_18(ip, op, parm) {\
- BITBLK128V32_18(ip, 0, op, parm);\
- BITBLK128V32_18(ip, 1, op, parm); IPPE(ip); OPPE(op += 18*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_19(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 5));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 11));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 3));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 9));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 1));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 7));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 13)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_19(ip, op, parm) {\
- BITBLK128V32_19(ip, 0, op, parm); IPPE(ip); OPPE(op += 19*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_20(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 5, iv), 4));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 12)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_20(ip, op, parm) {\
- BITBLK128V32_20(ip, 0, op, parm);\
- BITBLK128V32_20(ip, 1, op, parm);\
- BITBLK128V32_20(ip, 2, op, parm);\
- BITBLK128V32_20(ip, 3, op, parm); IPPE(ip); OPPE(op += 20*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_21(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 9));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 7));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 5));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 3));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 1));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 11)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_21(ip, op, parm) {\
- BITBLK128V32_21(ip, 0, op, parm); IPPE(ip); OPPE(op += 21*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_22(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 2));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 6));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 10)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_22(ip, op, parm) {\
- BITBLK128V32_22(ip, 0, op, parm);\
- BITBLK128V32_22(ip, 1, op, parm); IPPE(ip); OPPE(op += 22*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_23(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 5));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 1));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 7));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 3));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 9)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_23(ip, op, parm) {\
- BITBLK128V32_23(ip, 0, op, parm); IPPE(ip); OPPE(op += 23*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_24(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*4+ 1, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*4+ 2, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 3, iv), 8)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_24(ip, op, parm) {\
- BITBLK128V32_24(ip, 0, op, parm);\
- BITBLK128V32_24(ip, 1, op, parm);\
- BITBLK128V32_24(ip, 2, op, parm);\
- BITBLK128V32_24(ip, 3, op, parm);\
- BITBLK128V32_24(ip, 4, op, parm);\
- BITBLK128V32_24(ip, 5, op, parm);\
- BITBLK128V32_24(ip, 6, op, parm);\
- BITBLK128V32_24(ip, 7, op, parm); IPPE(ip); OPPE(op += 24*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_25(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 1));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 5));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 3));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 7)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_25(ip, op, parm) {\
- BITBLK128V32_25(ip, 0, op, parm); IPPE(ip); OPPE(op += 25*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_26(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 2));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 6)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_26(ip, op, parm) {\
- BITBLK128V32_26(ip, 0, op, parm);\
- BITBLK128V32_26(ip, 1, op, parm); IPPE(ip); OPPE(op += 26*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_27(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 1));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 3));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 5)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_27(ip, op, parm) {\
- BITBLK128V32_27(ip, 0, op, parm); IPPE(ip); OPPE(op += 27*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_28(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 4)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_28(ip, op, parm) {\
- BITBLK128V32_28(ip, 0, op, parm);\
- BITBLK128V32_28(ip, 1, op, parm);\
- BITBLK128V32_28(ip, 2, op, parm);\
- BITBLK128V32_28(ip, 3, op, parm); IPPE(ip); OPPE(op += 28*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_29(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 27);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 1));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 3)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_29(ip, op, parm) {\
- BITBLK128V32_29(ip, 0, op, parm); IPPE(ip); OPPE(op += 29*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_30(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 2)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_30(ip, op, parm) {\
- BITBLK128V32_30(ip, 0, op, parm);\
- BITBLK128V32_30(ip, 1, op, parm); IPPE(ip); OPPE(op += 30*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_31(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 27);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 3)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 29);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 2)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 30);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 1)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_31(ip, op, parm) {\
- BITBLK128V32_31(ip, 0, op, parm); IPPE(ip); OPPE(op += 31*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_32(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*1+ 0, iv, parm); ov = IP32(ip, i*1+ 0, iv); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_32(ip, op, parm) {\
- BITBLK128V32_32(ip, 0, op, parm);\
- BITBLK128V32_32(ip, 1, op, parm);\
- BITBLK128V32_32(ip, 2, op, parm);\
- BITBLK128V32_32(ip, 3, op, parm);\
- BITBLK128V32_32(ip, 4, op, parm);\
- BITBLK128V32_32(ip, 5, op, parm);\
- BITBLK128V32_32(ip, 6, op, parm);\
- BITBLK128V32_32(ip, 7, op, parm);\
- BITBLK128V32_32(ip, 8, op, parm);\
- BITBLK128V32_32(ip, 9, op, parm);\
- BITBLK128V32_32(ip, 10, op, parm);\
- BITBLK128V32_32(ip, 11, op, parm);\
- BITBLK128V32_32(ip, 12, op, parm);\
- BITBLK128V32_32(ip, 13, op, parm);\
- BITBLK128V32_32(ip, 14, op, parm);\
- BITBLK128V32_32(ip, 15, op, parm);\
- BITBLK128V32_32(ip, 16, op, parm);\
- BITBLK128V32_32(ip, 17, op, parm);\
- BITBLK128V32_32(ip, 18, op, parm);\
- BITBLK128V32_32(ip, 19, op, parm);\
- BITBLK128V32_32(ip, 20, op, parm);\
- BITBLK128V32_32(ip, 21, op, parm);\
- BITBLK128V32_32(ip, 22, op, parm);\
- BITBLK128V32_32(ip, 23, op, parm);\
- BITBLK128V32_32(ip, 24, op, parm);\
- BITBLK128V32_32(ip, 25, op, parm);\
- BITBLK128V32_32(ip, 26, op, parm);\
- BITBLK128V32_32(ip, 27, op, parm);\
- BITBLK128V32_32(ip, 28, op, parm);\
- BITBLK128V32_32(ip, 29, op, parm);\
- BITBLK128V32_32(ip, 30, op, parm);\
- BITBLK128V32_32(ip, 31, op, parm); IPPE(ip); OPPE(op += 32*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_1(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 1));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 3));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 5));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 7));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 9));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 11));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 13));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 15));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 17));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 18));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 19));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 21));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 22));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 23));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 25));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 26));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 27));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 28));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 29));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 30));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 31)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_1(ip, op, parm) {\
- BITBLK256V32_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_2(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 2));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 6));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 10));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 14));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 18));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 20));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 22));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 24));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 26));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 28));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 30)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_2(ip, op, parm) {\
- BITBLK256V32_2(ip, 0, op, parm);\
- BITBLK256V32_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_3(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 3));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 9));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 15));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 18));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 21));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 27));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 1));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 7));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 13));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 19));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 22));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 25));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 28));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 5));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 11));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 14));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 17));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 23));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 26));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 29)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_3(ip, op, parm) {\
- BITBLK256V32_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_4(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 1, iv), 4));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 3, iv), 12));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 5, iv), 20));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 6, iv), 24));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 28)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_4(ip, op, parm) {\
- BITBLK256V32_4(ip, 0, op, parm);\
- BITBLK256V32_4(ip, 1, op, parm);\
- BITBLK256V32_4(ip, 2, op, parm);\
- BITBLK256V32_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_5(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 5));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 15));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 20));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 25));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 3));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 13));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 18));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 23));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 1));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 11));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 21));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 26));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 9));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 14));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 19));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 7));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 17));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 22));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 27)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_5(ip, op, parm) {\
- BITBLK256V32_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_6(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 6));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 12));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 18));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 24));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 10));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 22));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 2));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 14));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 20));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 26)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_6(ip, op, parm) {\
- BITBLK256V32_6(ip, 0, op, parm);\
- BITBLK256V32_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_7(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 7));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 14));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 21));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 3));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 10));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 17));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 13));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 20));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 9));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 23));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 5));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 19));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 1));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 15));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 22));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 11));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 18));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 25)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_7(ip, op, parm) {\
- BITBLK256V32_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_8(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 1, iv), 8));\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 2, iv), 16));\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 3, iv), 24)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_8(ip, op, parm) {\
- BITBLK256V32_8(ip, 0, op, parm);\
- BITBLK256V32_8(ip, 1, op, parm);\
- BITBLK256V32_8(ip, 2, op, parm);\
- BITBLK256V32_8(ip, 3, op, parm);\
- BITBLK256V32_8(ip, 4, op, parm);\
- BITBLK256V32_8(ip, 5, op, parm);\
- BITBLK256V32_8(ip, 6, op, parm);\
- BITBLK256V32_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_9(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 9));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 18));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 13));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 22));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 17));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 3));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 21));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 7));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 11));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 15));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 1));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 10));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 19));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 5));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 14));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 23)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_9(ip, op, parm) {\
- BITBLK256V32_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_10(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 10));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 20));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 18));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 6));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 14));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 2));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 12));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 22)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_10(ip, op, parm) {\
- BITBLK256V32_10(ip, 0, op, parm);\
- BITBLK256V32_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_11(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 11));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 1));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 13));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 3));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 14));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 15));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 5));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 6));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 17));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 7));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 18));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 19));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 9));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 10));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 21)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_11(ip, op, parm) {\
- BITBLK256V32_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_12(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 1, iv), 12));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 3, iv), 4));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 6, iv), 8));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 20)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_12(ip, op, parm) {\
- BITBLK256V32_12(ip, 0, op, parm);\
- BITBLK256V32_12(ip, 1, op, parm);\
- BITBLK256V32_12(ip, 2, op, parm);\
- BITBLK256V32_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_13(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 13));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 7));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 1));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 14));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 15));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 9));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 3));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 10));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 17));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 11));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 5));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 18));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 6));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 19)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_13(ip, op, parm) {\
- BITBLK256V32_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_14(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 14));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 10));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 6));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 2));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 12));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 4));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 18)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_14(ip, op, parm) {\
- BITBLK256V32_14(ip, 0, op, parm);\
- BITBLK256V32_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_15(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 15));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 13));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 11));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 9));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 7));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 5));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 3));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 1));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 14));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 10));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 6));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 2));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 17)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_15(ip, op, parm) {\
- BITBLK256V32_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_16(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*2+ 0, iv, parm); ov = IP32(ip, i*2+ 0, iv);\
- VI32(ip, i*2+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*2+ 1, iv), 16)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_16(ip, op, parm) {\
- BITBLK256V32_16(ip, 0, op, parm);\
- BITBLK256V32_16(ip, 1, op, parm);\
- BITBLK256V32_16(ip, 2, op, parm);\
- BITBLK256V32_16(ip, 3, op, parm);\
- BITBLK256V32_16(ip, 4, op, parm);\
- BITBLK256V32_16(ip, 5, op, parm);\
- BITBLK256V32_16(ip, 6, op, parm);\
- BITBLK256V32_16(ip, 7, op, parm);\
- BITBLK256V32_16(ip, 8, op, parm);\
- BITBLK256V32_16(ip, 9, op, parm);\
- BITBLK256V32_16(ip, 10, op, parm);\
- BITBLK256V32_16(ip, 11, op, parm);\
- BITBLK256V32_16(ip, 12, op, parm);\
- BITBLK256V32_16(ip, 13, op, parm);\
- BITBLK256V32_16(ip, 14, op, parm);\
- BITBLK256V32_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_17(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 1));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 3));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 5));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 7));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 9));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 11));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 13));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 15)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_17(ip, op, parm) {\
- BITBLK256V32_17(ip, 0, op, parm); IPPE(ip); OPPE(op += 17*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_18(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 2));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 6));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 10));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 14)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_18(ip, op, parm) {\
- BITBLK256V32_18(ip, 0, op, parm);\
- BITBLK256V32_18(ip, 1, op, parm); IPPE(ip); OPPE(op += 18*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_19(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 5));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 11));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 3));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 9));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 1));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 7));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 13)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_19(ip, op, parm) {\
- BITBLK256V32_19(ip, 0, op, parm); IPPE(ip); OPPE(op += 19*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_20(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 5, iv), 4));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 12)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_20(ip, op, parm) {\
- BITBLK256V32_20(ip, 0, op, parm);\
- BITBLK256V32_20(ip, 1, op, parm);\
- BITBLK256V32_20(ip, 2, op, parm);\
- BITBLK256V32_20(ip, 3, op, parm); IPPE(ip); OPPE(op += 20*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_21(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 9));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 7));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 5));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 3));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 1));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 11)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_21(ip, op, parm) {\
- BITBLK256V32_21(ip, 0, op, parm); IPPE(ip); OPPE(op += 21*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_22(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 2));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 6));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 10)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_22(ip, op, parm) {\
- BITBLK256V32_22(ip, 0, op, parm);\
- BITBLK256V32_22(ip, 1, op, parm); IPPE(ip); OPPE(op += 22*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_23(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 5));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 1));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 7));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 3));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 9)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_23(ip, op, parm) {\
- BITBLK256V32_23(ip, 0, op, parm); IPPE(ip); OPPE(op += 23*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_24(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*4+ 1, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*4+ 2, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 3, iv), 8)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_24(ip, op, parm) {\
- BITBLK256V32_24(ip, 0, op, parm);\
- BITBLK256V32_24(ip, 1, op, parm);\
- BITBLK256V32_24(ip, 2, op, parm);\
- BITBLK256V32_24(ip, 3, op, parm);\
- BITBLK256V32_24(ip, 4, op, parm);\
- BITBLK256V32_24(ip, 5, op, parm);\
- BITBLK256V32_24(ip, 6, op, parm);\
- BITBLK256V32_24(ip, 7, op, parm); IPPE(ip); OPPE(op += 24*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_25(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 1));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 5));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 3));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 7)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_25(ip, op, parm) {\
- BITBLK256V32_25(ip, 0, op, parm); IPPE(ip); OPPE(op += 25*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_26(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 2));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 6)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_26(ip, op, parm) {\
- BITBLK256V32_26(ip, 0, op, parm);\
- BITBLK256V32_26(ip, 1, op, parm); IPPE(ip); OPPE(op += 26*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_27(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 1));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 3));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 5)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_27(ip, op, parm) {\
- BITBLK256V32_27(ip, 0, op, parm); IPPE(ip); OPPE(op += 27*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_28(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 4)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_28(ip, op, parm) {\
- BITBLK256V32_28(ip, 0, op, parm);\
- BITBLK256V32_28(ip, 1, op, parm);\
- BITBLK256V32_28(ip, 2, op, parm);\
- BITBLK256V32_28(ip, 3, op, parm); IPPE(ip); OPPE(op += 28*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_29(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 5)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 27);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 1));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 3)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_29(ip, op, parm) {\
- BITBLK256V32_29(ip, 0, op, parm); IPPE(ip); OPPE(op += 29*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_30(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 2)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_30(ip, op, parm) {\
- BITBLK256V32_30(ip, 0, op, parm);\
- BITBLK256V32_30(ip, 1, op, parm); IPPE(ip); OPPE(op += 30*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_31(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 5)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 27);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 3)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 29);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 2)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 30);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 1)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_31(ip, op, parm) {\
- BITBLK256V32_31(ip, 0, op, parm); IPPE(ip); OPPE(op += 31*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_32(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*1+ 0, iv, parm); ov = IP32(ip, i*1+ 0, iv); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_32(ip, op, parm) {\
- BITBLK256V32_32(ip, 0, op, parm);\
- BITBLK256V32_32(ip, 1, op, parm);\
- BITBLK256V32_32(ip, 2, op, parm);\
- BITBLK256V32_32(ip, 3, op, parm);\
- BITBLK256V32_32(ip, 4, op, parm);\
- BITBLK256V32_32(ip, 5, op, parm);\
- BITBLK256V32_32(ip, 6, op, parm);\
- BITBLK256V32_32(ip, 7, op, parm);\
- BITBLK256V32_32(ip, 8, op, parm);\
- BITBLK256V32_32(ip, 9, op, parm);\
- BITBLK256V32_32(ip, 10, op, parm);\
- BITBLK256V32_32(ip, 11, op, parm);\
- BITBLK256V32_32(ip, 12, op, parm);\
- BITBLK256V32_32(ip, 13, op, parm);\
- BITBLK256V32_32(ip, 14, op, parm);\
- BITBLK256V32_32(ip, 15, op, parm);\
- BITBLK256V32_32(ip, 16, op, parm);\
- BITBLK256V32_32(ip, 17, op, parm);\
- BITBLK256V32_32(ip, 18, op, parm);\
- BITBLK256V32_32(ip, 19, op, parm);\
- BITBLK256V32_32(ip, 20, op, parm);\
- BITBLK256V32_32(ip, 21, op, parm);\
- BITBLK256V32_32(ip, 22, op, parm);\
- BITBLK256V32_32(ip, 23, op, parm);\
- BITBLK256V32_32(ip, 24, op, parm);\
- BITBLK256V32_32(ip, 25, op, parm);\
- BITBLK256V32_32(ip, 26, op, parm);\
- BITBLK256V32_32(ip, 27, op, parm);\
- BITBLK256V32_32(ip, 28, op, parm);\
- BITBLK256V32_32(ip, 29, op, parm);\
- BITBLK256V32_32(ip, 30, op, parm);\
- BITBLK256V32_32(ip, 31, op, parm); IPPE(ip); OPPE(op += 32*4/sizeof(op[0]));\
-}
-
-//#include "bitpack_0.h"
-#define BITPACK128V16(_ip_, _nbits_, _op_, __parm) { __m128i *_ip=(__m128i *)_ip_,*_op=(__m128i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK128V16_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK128V16_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK128V16_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK128V16_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK128V16_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK128V16_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK128V16_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK128V16_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK128V16_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK128V16_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK128V16_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK128V16_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK128V16_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK128V16_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK128V16_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK128V16_16(_ip, _op, __parm); } break;\
- }\
-}
-
-#define BITPACK128V32(_ip_, _nbits_, _op_, __parm) { __m128i *_ip=(__m128i *)_ip_,*_op=(__m128i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK128V32_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK128V32_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK128V32_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK128V32_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK128V32_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK128V32_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK128V32_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK128V32_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK128V32_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK128V32_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK128V32_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK128V32_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK128V32_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK128V32_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK128V32_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK128V32_16(_ip, _op, __parm); } break;\
- case 17:{ BITPACK128V32_17(_ip, _op, __parm); } break;\
- case 18:{ BITPACK128V32_18(_ip, _op, __parm); } break;\
- case 19:{ BITPACK128V32_19(_ip, _op, __parm); } break;\
- case 20:{ BITPACK128V32_20(_ip, _op, __parm); } break;\
- case 21:{ BITPACK128V32_21(_ip, _op, __parm); } break;\
- case 22:{ BITPACK128V32_22(_ip, _op, __parm); } break;\
- case 23:{ BITPACK128V32_23(_ip, _op, __parm); } break;\
- case 24:{ BITPACK128V32_24(_ip, _op, __parm); } break;\
- case 25:{ BITPACK128V32_25(_ip, _op, __parm); } break;\
- case 26:{ BITPACK128V32_26(_ip, _op, __parm); } break;\
- case 27:{ BITPACK128V32_27(_ip, _op, __parm); } break;\
- case 28:{ BITPACK128V32_28(_ip, _op, __parm); } break;\
- case 29:{ BITPACK128V32_29(_ip, _op, __parm); } break;\
- case 30:{ BITPACK128V32_30(_ip, _op, __parm); } break;\
- case 31:{ BITPACK128V32_31(_ip, _op, __parm); } break;\
- case 32:{ BITPACK128V32_32(_ip, _op, __parm); } break;\
- }\
-}
-
-#define BITPACK256V32(_ip_, _nbits_, _op_, __parm) { __m256i *_ip=(__m256i *)_ip_,*_op=(__m256i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK256V32_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK256V32_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK256V32_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK256V32_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK256V32_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK256V32_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK256V32_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK256V32_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK256V32_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK256V32_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK256V32_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK256V32_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK256V32_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK256V32_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK256V32_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK256V32_16(_ip, _op, __parm); } break;\
- case 17:{ BITPACK256V32_17(_ip, _op, __parm); } break;\
- case 18:{ BITPACK256V32_18(_ip, _op, __parm); } break;\
- case 19:{ BITPACK256V32_19(_ip, _op, __parm); } break;\
- case 20:{ BITPACK256V32_20(_ip, _op, __parm); } break;\
- case 21:{ BITPACK256V32_21(_ip, _op, __parm); } break;\
- case 22:{ BITPACK256V32_22(_ip, _op, __parm); } break;\
- case 23:{ BITPACK256V32_23(_ip, _op, __parm); } break;\
- case 24:{ BITPACK256V32_24(_ip, _op, __parm); } break;\
- case 25:{ BITPACK256V32_25(_ip, _op, __parm); } break;\
- case 26:{ BITPACK256V32_26(_ip, _op, __parm); } break;\
- case 27:{ BITPACK256V32_27(_ip, _op, __parm); } break;\
- case 28:{ BITPACK256V32_28(_ip, _op, __parm); } break;\
- case 29:{ BITPACK256V32_29(_ip, _op, __parm); } break;\
- case 30:{ BITPACK256V32_30(_ip, _op, __parm); } break;\
- case 31:{ BITPACK256V32_31(_ip, _op, __parm); } break;\
- case 32:{ BITPACK256V32_32(_ip, _op, __parm); } break;\
- }\
-}
-
diff --git a/src/ext/for/bitunpack.c b/src/ext/for/bitunpack.c
deleted file mode 100644
index 802df40caad..00000000000
--- a/src/ext/for/bitunpack.c
+++ /dev/null
@@ -1,2433 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// "Integer Compression" Bit Packing
-#define BITUTIL_IN
-#define VINT_IN
-#include "conf.h"
-#include "bitutil.h"
-#include "bitpack.h"
-#include "vint.h"
-#include <string.h>
-#include <stdlib.h>
-#include <stdbool.h>
-
-#define PAD8(_x_) (((_x_)+7)/8)
-
-#pragma warning( disable : 4005)
-#pragma warning( disable : 4090)
-#pragma warning( disable : 4068)
-
-#pragma GCC push_options
-#pragma GCC optimize ("align-functions=16")
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunsequenced"
-
-#ifndef __AVX2__ //----------------------------------- Plain -------------------------------------------------------------------------------------------
-typedef unsigned char *(*BITUNPACK_F8)( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D8)( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start);
-typedef unsigned char *(*BITUNPACK_F16)(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D16)(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start);
-typedef unsigned char *(*BITUNPACK_F32)(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D32)(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start);
-typedef unsigned char *(*BITUNPACK_F64)(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D64)(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start);
-
- #if 0 //????
-#define OP(_op_, _x_) *_op_++
-#define OPX(_op_)
- #else
-#define OP(_op_, _x_) _op_[_x_]
-#define OPX(_op_) _op_ += 32
- #endif
-
-#define OPI(_op_,_nb_,_parm_) OPX(_op_)
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = _w_
-#define _BITUNPACK_ bitunpack
-#include "bitunpack_.h"
-
-#define DELTA
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += (_w_))
-#define _BITUNPACK_ bitdunpack // delta + 0
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += TEMPLATE2(zigzagdec, USIZE)(_w_))
-#define _BITUNPACK_ bitzunpack // zigzag
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ + (_w_))
-#define _BITUNPACK_ bitfunpack // for
-#include "bitunpack_.h"
-
-#define OPI(_op_,_nb_,_parm_) OPX(_op_); _parm_ += 32
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += (_w_)) + (_x_+1)
-#define _BITUNPACK_ bitd1unpack // delta + 1
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = _parm_ + (_w_)+(_x_+1)
-#define _BITUNPACK_ bitf1unpack // for + 1
-#include "bitunpack_.h"
-#undef OPI
-
-#define BITNUNPACK(in, n, out, _csize_, _usize_) {\
- unsigned char *ip = in;\
- for(op = out,out+=n; op < out;) { unsigned oplen = out - op,b; if(oplen > _csize_) oplen = _csize_; PREFETCH(ip+512,0);\
- b = *ip++; ip = TEMPLATE2(bitunpacka, _usize_)[b](ip, oplen, op);\
- op += oplen;\
- } \
- return ip - in;\
-}
-
-#define BITNDUNPACK(in, n, out, _csize_, _usize_, _bitunpacka_) { if(!n) return 0;\
- unsigned char *ip = in;\
- TEMPLATE2(vbxget, _usize_)(ip, start);\
- for(*out++ = start,--n,op = out; op != out+(n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpacka_, _usize_)[b](ip, _csize_, op, start); op += _csize_; start = op[-1];\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(_bitunpacka_, _usize_)[b](ip, n, op, start); }\
- return ip - in;\
-}
-
-size_t bitnunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op; BITNUNPACK(in, n, out, 128, 8); }
-size_t bitnunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op; BITNUNPACK(in, n, out, 128, 16); }
-size_t bitnunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; BITNUNPACK(in, n, out, 128, 32); }
-size_t bitnunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op; BITNUNPACK(in, n, out, 128, 64); }
-
-size_t bitndunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitdunpacka); }
-size_t bitndunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitdunpacka); }
-size_t bitndunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitdunpacka); }
-size_t bitndunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitdunpacka); }
-
-size_t bitnd1unpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitd1unpacka); }
-size_t bitnd1unpack16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitd1unpacka); }
-size_t bitnd1unpack32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitd1unpacka); }
-size_t bitnd1unpack64(unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitd1unpacka); }
-
-size_t bitnzunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitzunpacka); }
-size_t bitnzunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitzunpacka); }
-size_t bitnzunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitzunpacka); }
-size_t bitnzunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitzunpacka); }
-
-size_t bitnfunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitfunpacka); }
-size_t bitnfunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitfunpacka); }
-size_t bitnfunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitfunpacka); }
-size_t bitnfunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitfunpacka); }
-
-#endif
-//-------------------------------------------- SSE/AVX2 ---------------------------------------------------------------------------------------
-
-#define _BITNUNPACKV(in, n, out, _csize_, _usize_, _bitunpackv_) {\
- unsigned char *ip = in;\
- for(op = out; op != out+(n&~(_csize_-1)); op += _csize_) { PREFETCH(in+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpackv_, _usize_)(ip, _csize_, op,b);\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(bitunpack, _usize_)(ip, n, op,b); }\
- return ip - in;\
-}
-
-#define _BITNDUNPACKV(in, n, out, _csize_, _usize_, _bitunpackv_, _bitunpack_) { if(!n) return 0;\
- unsigned char *ip = in;\
- TEMPLATE2(vbxget, _usize_)(ip, start); \
- *out++ = start;\
- for(--n,op = out; op != out+(n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpackv_, _usize_)(ip, _csize_, op, start,b); op += _csize_; start = op[-1];\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(_bitunpack_, _usize_)(ip, n, op, start,b); }\
- return ip - in;\
-}
- #ifdef __AVX2__ //-------------------------------- AVX2 ----------------------------------------------------------------------------
-#include <immintrin.h>
-
- #ifdef __AVX512F__
-#define mm256_maskz_expand_epi32(_m_,_v_) _mm256_maskz_expand_epi32(_m_,_v_)
-#define mm256_maskz_loadu_epi32( _m_,_v_) _mm256_maskz_loadu_epi32( _m_,_v_)
- #else
-#if !(defined(_M_X64) || defined(__amd64__)) && (defined(__i386__) || defined(_M_IX86)) && !defined(__clang__)
-static inline __m128i _mm_cvtsi64_si128(__int64 a) { return _mm_loadl_epi64((__m128i*)&a); }
- #endif
-static ALIGNED(unsigned char, permv[256][8], 32) = {
-0,0,0,0,0,0,0,0,
-0,1,1,1,1,1,1,1,
-1,0,1,1,1,1,1,1,
-0,1,2,2,2,2,2,2,
-1,1,0,1,1,1,1,1,
-0,2,1,2,2,2,2,2,
-2,0,1,2,2,2,2,2,
-0,1,2,3,3,3,3,3,
-1,1,1,0,1,1,1,1,
-0,2,2,1,2,2,2,2,
-2,0,2,1,2,2,2,2,
-0,1,3,2,3,3,3,3,
-2,2,0,1,2,2,2,2,
-0,3,1,2,3,3,3,3,
-3,0,1,2,3,3,3,3,
-0,1,2,3,4,4,4,4,
-1,1,1,1,0,1,1,1,
-0,2,2,2,1,2,2,2,
-2,0,2,2,1,2,2,2,
-0,1,3,3,2,3,3,3,
-2,2,0,2,1,2,2,2,
-0,3,1,3,2,3,3,3,
-3,0,1,3,2,3,3,3,
-0,1,2,4,3,4,4,4,
-2,2,2,0,1,2,2,2,
-0,3,3,1,2,3,3,3,
-3,0,3,1,2,3,3,3,
-0,1,4,2,3,4,4,4,
-3,3,0,1,2,3,3,3,
-0,4,1,2,3,4,4,4,
-4,0,1,2,3,4,4,4,
-0,1,2,3,4,5,5,5,
-1,1,1,1,1,0,1,1,
-0,2,2,2,2,1,2,2,
-2,0,2,2,2,1,2,2,
-0,1,3,3,3,2,3,3,
-2,2,0,2,2,1,2,2,
-0,3,1,3,3,2,3,3,
-3,0,1,3,3,2,3,3,
-0,1,2,4,4,3,4,4,
-2,2,2,0,2,1,2,2,
-0,3,3,1,3,2,3,3,
-3,0,3,1,3,2,3,3,
-0,1,4,2,4,3,4,4,
-3,3,0,1,3,2,3,3,
-0,4,1,2,4,3,4,4,
-4,0,1,2,4,3,4,4,
-0,1,2,3,5,4,5,5,
-2,2,2,2,0,1,2,2,
-0,3,3,3,1,2,3,3,
-3,0,3,3,1,2,3,3,
-0,1,4,4,2,3,4,4,
-3,3,0,3,1,2,3,3,
-0,4,1,4,2,3,4,4,
-4,0,1,4,2,3,4,4,
-0,1,2,5,3,4,5,5,
-3,3,3,0,1,2,3,3,
-0,4,4,1,2,3,4,4,
-4,0,4,1,2,3,4,4,
-0,1,5,2,3,4,5,5,
-4,4,0,1,2,3,4,4,
-0,5,1,2,3,4,5,5,
-5,0,1,2,3,4,5,5,
-0,1,2,3,4,5,6,6,
-1,1,1,1,1,1,0,1,
-0,2,2,2,2,2,1,2,
-2,0,2,2,2,2,1,2,
-0,1,3,3,3,3,2,3,
-2,2,0,2,2,2,1,2,
-0,3,1,3,3,3,2,3,
-3,0,1,3,3,3,2,3,
-0,1,2,4,4,4,3,4,
-2,2,2,0,2,2,1,2,
-0,3,3,1,3,3,2,3,
-3,0,3,1,3,3,2,3,
-0,1,4,2,4,4,3,4,
-3,3,0,1,3,3,2,3,
-0,4,1,2,4,4,3,4,
-4,0,1,2,4,4,3,4,
-0,1,2,3,5,5,4,5,
-2,2,2,2,0,2,1,2,
-0,3,3,3,1,3,2,3,
-3,0,3,3,1,3,2,3,
-0,1,4,4,2,4,3,4,
-3,3,0,3,1,3,2,3,
-0,4,1,4,2,4,3,4,
-4,0,1,4,2,4,3,4,
-0,1,2,5,3,5,4,5,
-3,3,3,0,1,3,2,3,
-0,4,4,1,2,4,3,4,
-4,0,4,1,2,4,3,4,
-0,1,5,2,3,5,4,5,
-4,4,0,1,2,4,3,4,
-0,5,1,2,3,5,4,5,
-5,0,1,2,3,5,4,5,
-0,1,2,3,4,6,5,6,
-2,2,2,2,2,0,1,2,
-0,3,3,3,3,1,2,3,
-3,0,3,3,3,1,2,3,
-0,1,4,4,4,2,3,4,
-3,3,0,3,3,1,2,3,
-0,4,1,4,4,2,3,4,
-4,0,1,4,4,2,3,4,
-0,1,2,5,5,3,4,5,
-3,3,3,0,3,1,2,3,
-0,4,4,1,4,2,3,4,
-4,0,4,1,4,2,3,4,
-0,1,5,2,5,3,4,5,
-4,4,0,1,4,2,3,4,
-0,5,1,2,5,3,4,5,
-5,0,1,2,5,3,4,5,
-0,1,2,3,6,4,5,6,
-3,3,3,3,0,1,2,3,
-0,4,4,4,1,2,3,4,
-4,0,4,4,1,2,3,4,
-0,1,5,5,2,3,4,5,
-4,4,0,4,1,2,3,4,
-0,5,1,5,2,3,4,5,
-5,0,1,5,2,3,4,5,
-0,1,2,6,3,4,5,6,
-4,4,4,0,1,2,3,4,
-0,5,5,1,2,3,4,5,
-5,0,5,1,2,3,4,5,
-0,1,6,2,3,4,5,6,
-5,5,0,1,2,3,4,5,
-0,6,1,2,3,4,5,6,
-6,0,1,2,3,4,5,6,
-0,1,2,3,4,5,6,7,
-1,1,1,1,1,1,1,0,
-0,2,2,2,2,2,2,1,
-2,0,2,2,2,2,2,1,
-0,1,3,3,3,3,3,2,
-2,2,0,2,2,2,2,1,
-0,3,1,3,3,3,3,2,
-3,0,1,3,3,3,3,2,
-0,1,2,4,4,4,4,3,
-2,2,2,0,2,2,2,1,
-0,3,3,1,3,3,3,2,
-3,0,3,1,3,3,3,2,
-0,1,4,2,4,4,4,3,
-3,3,0,1,3,3,3,2,
-0,4,1,2,4,4,4,3,
-4,0,1,2,4,4,4,3,
-0,1,2,3,5,5,5,4,
-2,2,2,2,0,2,2,1,
-0,3,3,3,1,3,3,2,
-3,0,3,3,1,3,3,2,
-0,1,4,4,2,4,4,3,
-3,3,0,3,1,3,3,2,
-0,4,1,4,2,4,4,3,
-4,0,1,4,2,4,4,3,
-0,1,2,5,3,5,5,4,
-3,3,3,0,1,3,3,2,
-0,4,4,1,2,4,4,3,
-4,0,4,1,2,4,4,3,
-0,1,5,2,3,5,5,4,
-4,4,0,1,2,4,4,3,
-0,5,1,2,3,5,5,4,
-5,0,1,2,3,5,5,4,
-0,1,2,3,4,6,6,5,
-2,2,2,2,2,0,2,1,
-0,3,3,3,3,1,3,2,
-3,0,3,3,3,1,3,2,
-0,1,4,4,4,2,4,3,
-3,3,0,3,3,1,3,2,
-0,4,1,4,4,2,4,3,
-4,0,1,4,4,2,4,3,
-0,1,2,5,5,3,5,4,
-3,3,3,0,3,1,3,2,
-0,4,4,1,4,2,4,3,
-4,0,4,1,4,2,4,3,
-0,1,5,2,5,3,5,4,
-4,4,0,1,4,2,4,3,
-0,5,1,2,5,3,5,4,
-5,0,1,2,5,3,5,4,
-0,1,2,3,6,4,6,5,
-3,3,3,3,0,1,3,2,
-0,4,4,4,1,2,4,3,
-4,0,4,4,1,2,4,3,
-0,1,5,5,2,3,5,4,
-4,4,0,4,1,2,4,3,
-0,5,1,5,2,3,5,4,
-5,0,1,5,2,3,5,4,
-0,1,2,6,3,4,6,5,
-4,4,4,0,1,2,4,3,
-0,5,5,1,2,3,5,4,
-5,0,5,1,2,3,5,4,
-0,1,6,2,3,4,6,5,
-5,5,0,1,2,3,5,4,
-0,6,1,2,3,4,6,5,
-6,0,1,2,3,4,6,5,
-0,1,2,3,4,5,7,6,
-2,2,2,2,2,2,0,1,
-0,3,3,3,3,3,1,2,
-3,0,3,3,3,3,1,2,
-0,1,4,4,4,4,2,3,
-3,3,0,3,3,3,1,2,
-0,4,1,4,4,4,2,3,
-4,0,1,4,4,4,2,3,
-0,1,2,5,5,5,3,4,
-3,3,3,0,3,3,1,2,
-0,4,4,1,4,4,2,3,
-4,0,4,1,4,4,2,3,
-0,1,5,2,5,5,3,4,
-4,4,0,1,4,4,2,3,
-0,5,1,2,5,5,3,4,
-5,0,1,2,5,5,3,4,
-0,1,2,3,6,6,4,5,
-3,3,3,3,0,3,1,2,
-0,4,4,4,1,4,2,3,
-4,0,4,4,1,4,2,3,
-0,1,5,5,2,5,3,4,
-4,4,0,4,1,4,2,3,
-0,5,1,5,2,5,3,4,
-5,0,1,5,2,5,3,4,
-0,1,2,6,3,6,4,5,
-4,4,4,0,1,4,2,3,
-0,5,5,1,2,5,3,4,
-5,0,5,1,2,5,3,4,
-0,1,6,2,3,6,4,5,
-5,5,0,1,2,5,3,4,
-0,6,1,2,3,6,4,5,
-6,0,1,2,3,6,4,5,
-0,1,2,3,4,7,5,6,
-3,3,3,3,3,0,1,2,
-0,4,4,4,4,1,2,3,
-4,0,4,4,4,1,2,3,
-0,1,5,5,5,2,3,4,
-4,4,0,4,4,1,2,3,
-0,5,1,5,5,2,3,4,
-5,0,1,5,5,2,3,4,
-0,1,2,6,6,3,4,5,
-4,4,4,0,4,1,2,3,
-0,5,5,1,5,2,3,4,
-5,0,5,1,5,2,3,4,
-0,1,6,2,6,3,4,5,
-5,5,0,1,5,2,3,4,
-0,6,1,2,6,3,4,5,
-6,0,1,2,6,3,4,5,
-0,1,2,3,7,4,5,6,
-4,4,4,4,0,1,2,3,
-0,5,5,5,1,2,3,4,
-5,0,5,5,1,2,3,4,
-0,1,6,6,2,3,4,5,
-5,5,0,5,1,2,3,4,
-0,6,1,6,2,3,4,5,
-6,0,1,6,2,3,4,5,
-0,1,2,7,3,4,5,6,
-5,5,5,0,1,2,3,4,
-0,6,6,1,2,3,4,5,
-6,0,6,1,2,3,4,5,
-0,1,7,2,3,4,5,6,
-6,6,0,1,2,3,4,5,
-0,7,1,2,3,4,5,6,
-7,0,1,2,3,4,5,6,
-0,1,2,3,4,5,6,7
-};
-#define u2vmask(_m_,_tv_) _mm256_sllv_epi32(_mm256_set1_epi8(_m_), _tv_)
-#define mm256_maskz_expand_epi32(_m_, _v_) _mm256_permutevar8x32_epi32(_v_, _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(ctou64(permv[_m_]))) )
-#define mm256_maskz_loadu_epi32(_m_,_v_) _mm256_blendv_epi8(zv, mm256_maskz_expand_epi32(xm, _mm256_loadu_si256((__m256i*)pex)), u2vmask(xm,tv)) // emulate AVX512 _mm256_maskz_loadu_epi32 on AVX2
- #endif
-
-//-----------------------------------------------------------------------------
-#define VO32( _op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, ov)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_)
-#include "bitunpack_.h"
-
-#define BITUNBLK256V32_0(ip, _i_, _op_, _nb_,_parm_) {__m256i ov;\
- VOZ32(_op_, 0, ov, _nb_,_parm_);\
- VOZ32(_op_, 1, ov, _nb_,_parm_);\
- VOZ32(_op_, 2, ov, _nb_,_parm_);\
- VOZ32(_op_, 3, ov, _nb_,_parm_);\
- VOZ32(_op_, 4, ov, _nb_,_parm_);\
- VOZ32(_op_, 5, ov, _nb_,_parm_);\
- VOZ32(_op_, 6, ov, _nb_,_parm_);\
- VOZ32(_op_, 7, ov, _nb_,_parm_);\
- VOZ32(_op_, 8, ov, _nb_,_parm_);\
- VOZ32(_op_, 9, ov, _nb_,_parm_);\
- VOZ32(_op_, 10, ov, _nb_,_parm_);\
- VOZ32(_op_, 11, ov, _nb_,_parm_);\
- VOZ32(_op_, 12, ov, _nb_,_parm_);\
- VOZ32(_op_, 13, ov, _nb_,_parm_);\
- VOZ32(_op_, 14, ov, _nb_,_parm_);\
- VOZ32(_op_, 15, ov, _nb_,_parm_);\
- VOZ32(_op_, 16, ov, _nb_,_parm_);\
- VOZ32(_op_, 17, ov, _nb_,_parm_);\
- VOZ32(_op_, 18, ov, _nb_,_parm_);\
- VOZ32(_op_, 19, ov, _nb_,_parm_);\
- VOZ32(_op_, 20, ov, _nb_,_parm_);\
- VOZ32(_op_, 21, ov, _nb_,_parm_);\
- VOZ32(_op_, 22, ov, _nb_,_parm_);\
- VOZ32(_op_, 23, ov, _nb_,_parm_);\
- VOZ32(_op_, 24, ov, _nb_,_parm_);\
- VOZ32(_op_, 25, ov, _nb_,_parm_);\
- VOZ32(_op_, 26, ov, _nb_,_parm_);\
- VOZ32(_op_, 27, ov, _nb_,_parm_);\
- VOZ32(_op_, 28, ov, _nb_,_parm_);\
- VOZ32(_op_, 29, ov, _nb_,_parm_);\
- VOZ32(_op_, 30, ov, _nb_,_parm_);\
- VOZ32(_op_, 31, ov, _nb_,_parm_);\
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm256_setzero_si256()
-
-unsigned char *bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv;
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-//--------------------------------------- zeromask unpack for TurboPFor vp4d.c --------------------------------------
-#define VO32(_op_, _i_, _ov_, _nb_,_parm_) xm = *bb++; _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) )); pex += popcnt32(xm)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_parm_) xm = *bb++; _mm256_storeu_si256(_op_++, mm256_maskz_loadu_epi32(xm,(__m256i*)pex) ); pex += popcnt32(xm)
-#define BITUNPACK0(_parm_)
-#include "bitunpack_.h"
-unsigned char *_bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm; __m256i sv, zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_)
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);//, zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);// zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_parm_) _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, sv))
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitfunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-//-----------------------------------------------------------------------------
-#define VX32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) ); pex += popcnt32(xm)
-#define VXZ32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = mm256_maskz_loadu_epi32(xm,(__m256i*)pex); pex += popcnt32(xm)
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm;
- __m256i sv = _mm256_set1_epi32(start), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7),zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VX32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) ); pex += popcnt32(xm)
-#define VXZ32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = mm256_maskz_loadu_epi32(xm,(__m256i*)pex); pex += popcnt32(xm)
-
- #ifdef _MSC_VER
-#define SCAN32x8( _v_, _sv_) {\
- _v_ = _mm256_add_epi32(_v_, _mm256_slli_si256(_v_, 4));\
- _v_ = _mm256_add_epi32(_v_, _mm256_slli_si256(_v_, 8));\
- _sv_ = _mm256_add_epi32( _mm256_permute2x128_si256( _mm256_shuffle_epi32(_sv_,_MM_SHUFFLE(3, 3, 3, 3)), _sv_, 0x11), \
- _mm256_add_epi32(_v_, _mm256_permute2x128_si256(zv,_mm256_shuffle_epi32(_v_, _MM_SHUFFLE(3, 3, 3, 3)), 0x20)));\
-}
-#define SCANI32x8(_v_, _sv_, _vi_) SCAN32x8(_v_, _sv_); _sv_ = _mm256_add_epi32(_sv_, _vi_)
-#define ZIGZAG32x8(_v_) _mm256_xor_si256(_mm256_slli_epi32(_v_,1), _mm256_srai_epi32(_v_,31))
-#define UNZIGZAG32x8(_v_) _mm256_xor_si256(_mm256_srli_epi32(_v_,1), _mm256_srai_epi32(_mm256_slli_epi32(_v_,31),31) )
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _ov_ = UNZIGZAG32x8(_ov_); SCAN32x8(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _ov_ = UNZIGZAG32x8(_ov_); SCAN32x8(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
- #else
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
- #endif
-
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b);
- unsigned xm;
- const __m256i zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_); _parm_ = _mm256_add_epi32(_parm_, cv)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) _parm_ = _mm256_add_epi32(_parm_, cv); cv = _mm256_set1_epi32(8)
-unsigned char *bitd1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- const __m256i zv = _mm256_setzero_si256();
- __m256i sv = _mm256_set1_epi32(start), cv = _mm256_set_epi32(8,7,6,5,4,3,2,1);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, _sv_)); _sv_ = _mm256_add_epi32(_sv_, cv)
-#define VOZ32(_op_, _i_, ov, _nb_,_sv_) _mm256_storeu_si256(_op_++, _sv_); _sv_ = _mm256_add_epi32(_sv_, cv);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitf1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- const __m256i cv = _mm256_set1_epi32(8);
- __m256i sv = _mm256_set_epi32(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) mv = _mm256_set1_epi32(0) //_parm_ = _mm_setzero_si128()
-unsigned char *_bitd1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm;
- const __m256i cv = _mm256_set_epi32(8,7,6,5,4,3,2,1), zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-size_t bitnunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; _BITNUNPACKV( in, n, out, 256, 32, bitunpack256v); }
-size_t bitndunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitdunpack256v, bitdunpack); }
-size_t bitnd1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitd1unpack256v, bitd1unpack); }
-//size_t bitns1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bits1unpack256v, bitd1unpack); }
-size_t bitnzunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitzunpack256v, bitzunpack); }
-size_t bitnfunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitfunpack256v, bitfunpack); }
- #elif defined(__SSE2__) || defined(__ARM_NEON) //------------------------------ SSE2/SSSE3 ---------------------------------------------------------
-#define BITMAX16 16
-#define BITMAX32 32
-
-#define VO16( _op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, ov)
-#define VO32( _op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, ov)
-#include "bitunpack_.h"
-
-#define VOZ16(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define BITUNBLK128V16_0(ip, _i_, _op_, _nb_,_parm_) {__m128i ov;\
- VOZ16(_op_, 0, ov, _nb_,_parm_);\
- VOZ16(_op_, 1, ov, _nb_,_parm_);\
- VOZ16(_op_, 2, ov, _nb_,_parm_);\
- VOZ16(_op_, 3, ov, _nb_,_parm_);\
- VOZ16(_op_, 4, ov, _nb_,_parm_);\
- VOZ16(_op_, 5, ov, _nb_,_parm_);\
- VOZ16(_op_, 6, ov, _nb_,_parm_);\
- VOZ16(_op_, 7, ov, _nb_,_parm_);\
- VOZ16(_op_, 8, ov, _nb_,_parm_);\
- VOZ16(_op_, 9, ov, _nb_,_parm_);\
- VOZ16(_op_, 10, ov, _nb_,_parm_);\
- VOZ16(_op_, 11, ov, _nb_,_parm_);\
- VOZ16(_op_, 12, ov, _nb_,_parm_);\
- VOZ16(_op_, 13, ov, _nb_,_parm_);\
- VOZ16(_op_, 14, ov, _nb_,_parm_);\
- VOZ16(_op_, 15, ov, _nb_,_parm_);\
- /*VOZ16(_op_, 16, ov, _nb_,_parm_);\
- VOZ16(_op_, 17, ov, _nb_,_parm_);\
- VOZ16(_op_, 18, ov, _nb_,_parm_);\
- VOZ16(_op_, 19, ov, _nb_,_parm_);\
- VOZ16(_op_, 20, ov, _nb_,_parm_);\
- VOZ16(_op_, 21, ov, _nb_,_parm_);\
- VOZ16(_op_, 22, ov, _nb_,_parm_);\
- VOZ16(_op_, 23, ov, _nb_,_parm_);\
- VOZ16(_op_, 24, ov, _nb_,_parm_);\
- VOZ16(_op_, 25, ov, _nb_,_parm_);\
- VOZ16(_op_, 26, ov, _nb_,_parm_);\
- VOZ16(_op_, 27, ov, _nb_,_parm_);\
- VOZ16(_op_, 28, ov, _nb_,_parm_);\
- VOZ16(_op_, 29, ov, _nb_,_parm_);\
- VOZ16(_op_, 30, ov, _nb_,_parm_);\
- VOZ16(_op_, 31, ov, _nb_,_parm_);*/\
-}
-
-#define BITUNBLK128V32_0(ip, _i_, _op_, _nb_,_parm_) {__m128i ov;\
- VOZ32(_op_, 0, ov, _nb_,_parm_);\
- VOZ32(_op_, 1, ov, _nb_,_parm_);\
- VOZ32(_op_, 2, ov, _nb_,_parm_);\
- VOZ32(_op_, 3, ov, _nb_,_parm_);\
- VOZ32(_op_, 4, ov, _nb_,_parm_);\
- VOZ32(_op_, 5, ov, _nb_,_parm_);\
- VOZ32(_op_, 6, ov, _nb_,_parm_);\
- VOZ32(_op_, 7, ov, _nb_,_parm_);\
- VOZ32(_op_, 8, ov, _nb_,_parm_);\
- VOZ32(_op_, 9, ov, _nb_,_parm_);\
- VOZ32(_op_, 10, ov, _nb_,_parm_);\
- VOZ32(_op_, 11, ov, _nb_,_parm_);\
- VOZ32(_op_, 12, ov, _nb_,_parm_);\
- VOZ32(_op_, 13, ov, _nb_,_parm_);\
- VOZ32(_op_, 14, ov, _nb_,_parm_);\
- VOZ32(_op_, 15, ov, _nb_,_parm_);\
- VOZ32(_op_, 16, ov, _nb_,_parm_);\
- VOZ32(_op_, 17, ov, _nb_,_parm_);\
- VOZ32(_op_, 18, ov, _nb_,_parm_);\
- VOZ32(_op_, 19, ov, _nb_,_parm_);\
- VOZ32(_op_, 20, ov, _nb_,_parm_);\
- VOZ32(_op_, 21, ov, _nb_,_parm_);\
- VOZ32(_op_, 22, ov, _nb_,_parm_);\
- VOZ32(_op_, 23, ov, _nb_,_parm_);\
- VOZ32(_op_, 24, ov, _nb_,_parm_);\
- VOZ32(_op_, 25, ov, _nb_,_parm_);\
- VOZ32(_op_, 26, ov, _nb_,_parm_);\
- VOZ32(_op_, 27, ov, _nb_,_parm_);\
- VOZ32(_op_, 28, ov, _nb_,_parm_);\
- VOZ32(_op_, 29, ov, _nb_,_parm_);\
- VOZ32(_op_, 30, ov, _nb_,_parm_);\
- VOZ32(_op_, 31, ov, _nb_,_parm_);\
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm_setzero_si128()
-
-unsigned char *bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b) { const unsigned char *ip = in+PAD8(128*b); __m128i sv; BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip; }
-unsigned char *bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) { const unsigned char *ip = in+PAD8(128*b); __m128i sv; BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip; }
-unsigned char *bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) {
- const unsigned char *_in=in; unsigned *_out=out; __m128i sv;
- BITUNPACK128V32(in, b, out, sv); out = _out+128; in=_in+PAD8(128*b);
- BITUNPACK128V32(in, b, out, sv);
- return (unsigned char *)_in+PAD8(256*b);
-}
-static void applyException_8bits(uint8_t xm8, uint32_t** pPEX, int nb, uint32_t ov[8]) {
- uint32_t* ex = *pPEX;
- for (int j = 0; j < 8; j++) {
- if ((xm8 >> j) & 1) {
- ov[j] += (ex[0] << nb);
- ex++;
- }
- }
- *pPEX = ex;
-}
-static inline uint32_t zigzagDecode_scalar(uint32_t x) {
- // (x>>1) ^ -((x & 1) )
- return (x >> 1) ^ -(x & 1);
-}
-static void bitunblk256v32_scalar_template(uint32_t** pIn, uint32_t** pOut, int expansions_count,
- const uint8_t* SHIFT_HI, const uint8_t* SHIFT_LO,
- const uint8_t* READ_FLAG, uint32_t mask, int nb,
- bool isZigZag) {
- const uint32_t* oldp = NULL; // pointer to current block data
- uint32_t ov[8], tmp[8];
-
- for (int k = 0; k < expansions_count; k++) {
- if (k == 0) {
- // Step 0: Load input block and directly take the lower nb bits
- oldp = *pIn;
- *pIn += 8;
- for (int j = 0; j < 8; j++) {
- ov[j] = oldp[j] & mask;
- }
- } else {
- // First right shift the current block data by SHIFT_HI[k]
- for (int j = 0; j < 8; j++) {
- ov[j] = oldp[j] >> SHIFT_HI[k];
- }
- if (READ_FLAG[k]) {
- // Need to load a new block: left shift the new block data by SHIFT_LO[k], then merge with ov
- const uint32_t* newp = *pIn;
- *pIn += 8;
- for (int j = 0; j < 8; j++) {
- uint32_t part_lo = (newp[j] << SHIFT_LO[k]) & mask;
- ov[j] |= part_lo;
- }
- // Update current block pointer
- oldp = newp;
- } else {
- // No need to load a new block, ensure the result is within mask range
- for (int j = 0; j < 8; j++) {
- ov[j] &= mask;
- }
- }
- }
- // Write out the current 8 results
- uint32_t* outp = *pOut;
- for (int j = 0; j < 8; j++) {
- if (isZigZag) {
- outp[j] = zigzagDecode_scalar(ov[j]);
- } else {
- outp[j] = ov[j];
- }
- }
- *pOut += 8;
- }
-}
-/**
- * Generic template: supports "some expansions don't need to read new blocks".
- *
- * Parameters:
- * - expansions_count: total number of expansions (for 29-bit, it might be 32 times)
- * - SHIFT_HI[k], SHIFT_LO[k]: right shift for leftover, left shift for new block in k-th expansion
- * - READ_FLAG[k]: whether k-th expansion needs to read a new block (1 means yes, 0 means no)
- * - mask: for 29-bit = (1u << 29) - 1
- * - nb: base bits (29)
- */
-static void bitunblk256v32_scalarBlock_ex_template(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, int expansions_count,
- const uint8_t* SHIFT_HI, const uint8_t* SHIFT_LO,
- const uint8_t* READ_FLAG, uint32_t mask, int nb,
- bool isZigZag) {
- const uint32_t* oldp = NULL; // leftover block (previous batch)
-
- for (int k = 0; k < expansions_count; k++) {
- uint32_t ov[8];
-
- if (k == 0) {
- // First time: directly read 8×32-bit and apply mask
- oldp = *pIn;
- *pIn += 8;
- for (int j = 0; j < 8; j++) {
- ov[j] = oldp[j] & mask;
- }
- } else {
- // Subsequent expansions
- uint8_t hi = SHIFT_HI[k];
- uint8_t lo = SHIFT_LO[k];
-
- // First shift leftover >> hi
- for (int j = 0; j < 8; j++) {
- ov[j] = (oldp[j] >> hi);
- }
-
- // If this expansion needs to read a new block, append newp << lo
- if (READ_FLAG[k]) {
- const uint32_t* newp = *pIn;
- *pIn += 8;
- for (int j = 0; j < 8; j++) {
- uint32_t part_lo = (newp[j] << lo) & mask;
- ov[j] |= part_lo;
- }
- // After reading, newp becomes the leftover for next time
- oldp = newp;
- } else {
- // No need to read new block => just apply mask to leftover >> hi
- for (int j = 0; j < 8; j++) {
- ov[j] &= mask;
- }
- // leftover remains unchanged, continue using oldp
- }
- }
-
- // Apply exceptions
- uint8_t xm8 = **pBB;
- (*pBB)++;
- applyException_8bits(xm8, pPEX, nb, ov);
-
- // Write out this batch of 8 results
- uint32_t* outp = *pOut;
- for (int j = 0; j < 8; j++) {
- if (isZigZag) {
- outp[j] = zigzagDecode_scalar(ov[j]);
- } else {
- outp[j] = ov[j];
- }
- }
- *pOut += 8;
- }
-}
-static void bitunpack256v32_0_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- uint32_t* op = *pOut;
- for (int i = 0; i < 32; i++) {
- // Read bitmap if exists, otherwise default to 0
- uint8_t xm8 = (pBB != NULL) ? **pBB : 0;
- if (pBB != NULL) {
- (*pBB)++;
- }
- // Initialize output array (all zeros by default)
- uint32_t ov[8] = {0};
- if (xm8 != 0 && pPEX != NULL) {
- applyException_8bits(xm8, pPEX, 0, ov);
- }
-
- // Directly write 8 values using a loop to avoid repeated memory copy calls
- for (int j = 0; j < 8; j++) {
- if (isZigZag) {
- op[j] = zigzagDecode_scalar(ov[j]);
- } else {
- op[j] = ov[j];
- }
- }
- op += 8;
- }
- *pOut = op;
-}
-
-static void bitunpack256v32_1_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb1 = 1;
- const uint32_t mask1 = 1; // 0x1
- const int expansions_count_1 = 32;
- static const uint8_t SHIFT_HI_1[32] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
- 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
- 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
- static const uint8_t SHIFT_LO_1[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_1[32] = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_1, SHIFT_HI_1,
- SHIFT_LO_1, READ_FLAG_1, mask1, nb1, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_1, SHIFT_HI_1, SHIFT_LO_1,
- READ_FLAG_1, mask1, nb1, isZigZag);
- }
-}
-
-static void bitunpack256v32_2_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb2 = 2;
- const uint32_t mask2 = (1u << nb2) - 1; // 0x3
- const int expansions_count_2 = 16;
- static const uint8_t SHIFT_HI_2[16] = {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30};
- static const uint8_t SHIFT_LO_2[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_2[16] = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_2,
- SHIFT_HI_2, SHIFT_LO_2, READ_FLAG_2, mask2, nb2,
- isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_2, SHIFT_HI_2, SHIFT_LO_2,
- READ_FLAG_2, mask2, nb2, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_3_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb3 = 3;
- const uint32_t mask3 = (1u << nb3) - 1; // 0x7
- const int expansions_count_3 = 32;
- static const uint8_t SHIFT_HI_3[32] = {0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30,
- 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31,
- 2, 5, 8, 11, 14, 17, 20, 23, 26, 29};
- static const uint8_t SHIFT_LO_3[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_3[32] = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_3, SHIFT_HI_3,
- SHIFT_LO_3, READ_FLAG_3, mask3, nb3, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_3, SHIFT_HI_3, SHIFT_LO_3,
- READ_FLAG_3, mask3, nb3, isZigZag);
- }
-}
-
-static void bitunpack256v32_4_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const uint32_t mask4 = (1u << 4) - 1; // 0xF
- const int nb = 4; // base bits
- const int expansions_count = 8;
- static const uint8_t SHIFT_HI_4[8] = {0, 4, 8, 12, 16, 20, 24, 28};
- static const uint8_t SHIFT_LO_4[8] = {0, 0, 0, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_4[8] = {1, 0, 0, 0, 0, 0, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count, SHIFT_HI_4,
- SHIFT_LO_4, READ_FLAG_4, mask4, nb, isZigZag);
- }
- } else {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI_4, SHIFT_LO_4,
- READ_FLAG_4, mask4, nb, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_5_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb5 = 5;
- const uint32_t mask5 = (1u << nb5) - 1; // 0x1F
- const int expansions_count_5 = 32;
- static const uint8_t SHIFT_HI_5[32] = {0, 5, 10, 15, 20, 25, 30, 3, 8, 13, 18,
- 23, 28, 1, 6, 11, 16, 21, 26, 31, 4, 9,
- 14, 19, 24, 29, 2, 7, 12, 17, 22, 27};
- static const uint8_t SHIFT_LO_5[32] = {0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0,
- 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_5[32] = {1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_5, SHIFT_HI_5,
- SHIFT_LO_5, READ_FLAG_5, mask5, nb5, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_5, SHIFT_HI_5, SHIFT_LO_5,
- READ_FLAG_5, mask5, nb5, isZigZag);
- }
-}
-static void bitunpack256v32_6_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb6 = 6;
- const uint32_t mask6 = (1u << nb6) - 1; // 0x3F
- const int expansions_count_6 = 16;
- static const uint8_t SHIFT_HI_6[16] = {0, 6, 12, 18, 24, 30, 4, 10, 16, 22, 28, 2, 8, 14, 20, 26};
- static const uint8_t SHIFT_LO_6[16] = {0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_6[16] = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_6,
- SHIFT_HI_6, SHIFT_LO_6, READ_FLAG_6, mask6, nb6,
- isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_6, SHIFT_HI_6, SHIFT_LO_6,
- READ_FLAG_6, mask6, nb6, isZigZag);
- }
- }
-}
-static void bitunpack256v32_7_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb7 = 7;
- const uint32_t mask7 = (1u << nb7) - 1; // 0x7F
- const int expansions_count = 32;
- static const uint8_t SHIFT_HI_7[32] = {0, 7, 14, 21, 28, 3, 10, 17, 24, 31, 6,
- 13, 20, 27, 2, 9, 16, 23, 30, 5, 12, 19,
- 26, 1, 8, 15, 22, 29, 4, 11, 18, 25};
- static const uint8_t SHIFT_LO_7[32] = {0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0,
- 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0};
- static const uint8_t READ_FLAG_7[32] = {1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
- 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count, SHIFT_HI_7,
- SHIFT_LO_7, READ_FLAG_7, mask7, nb7, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI_7, SHIFT_LO_7,
- READ_FLAG_7, mask7, nb7, isZigZag);
- }
-}
-static void bitunpack256v32_8_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb8 = 8;
- const uint32_t mask8 = (1u << nb8) - 1; // 0xFF
- const int expansions_count_8 = 4;
- static const uint8_t SHIFT_HI_8[4] = {0, 8, 16, 24};
- static const uint8_t SHIFT_LO_8[4] = {0, 0, 0, 0};
- static const uint8_t READ_FLAG_8[4] = {1, 0, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 8; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_8,
- SHIFT_HI_8, SHIFT_LO_8, READ_FLAG_8, mask8, nb8,
- isZigZag);
- }
- } else {
- for (int i = 0; i < 8; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_8, SHIFT_HI_8, SHIFT_LO_8,
- READ_FLAG_8, mask8, nb8, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_9_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb9 = 9;
- const uint32_t mask9 = (1u << nb9) - 1; // 0x1FF
- const int expansions_count_9 = 32;
- static const uint8_t SHIFT_HI_9[32] = {0, 9, 18, 27, 4, 13, 22, 31, 8, 17, 26,
- 3, 12, 21, 30, 7, 16, 25, 2, 11, 20, 29,
- 6, 15, 24, 1, 10, 19, 28, 5, 14, 23};
- static const uint8_t SHIFT_LO_9[32] = {0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 2, 0,
- 0, 7, 0, 0, 0, 3, 0, 0, 8, 0, 0, 0, 4, 0, 0, 0};
- static const uint8_t READ_FLAG_9[32] = {1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0,
- 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_9, SHIFT_HI_9,
- SHIFT_LO_9, READ_FLAG_9, mask9, nb9, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_9, SHIFT_HI_9, SHIFT_LO_9,
- READ_FLAG_9, mask9, nb9, isZigZag);
- }
-}
-static void bitunpack256v32_10_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb10 = 10;
- const uint32_t mask10 = (1u << nb10) - 1; // 0x3FF
- const int expansions_count_10 = 16;
- static const uint8_t SHIFT_HI_10[16] = {0, 10, 20, 30, 8, 18, 28, 6,
- 16, 26, 4, 14, 24, 2, 12, 22};
- static const uint8_t SHIFT_LO_10[16] = {0, 0, 0, 2, 0, 0, 4, 0, 0, 6, 0, 0, 8, 0, 0, 0};
- static const uint8_t READ_FLAG_10[16] = {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_10,
- SHIFT_HI_10, SHIFT_LO_10, READ_FLAG_10, mask10,
- nb10, isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_10, SHIFT_HI_10, SHIFT_LO_10,
- READ_FLAG_10, mask10, nb10, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_11_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb11 = 11;
- const uint32_t mask11 = (1u << nb11) - 1; // 0x7FF
- const int expansions_count_11 = 32;
- static const uint8_t SHIFT_HI_11[32] = {0, 11, 22, 1, 12, 23, 2, 13, 24, 3, 14,
- 25, 4, 15, 26, 5, 16, 27, 6, 17, 28, 7,
- 18, 29, 8, 19, 30, 9, 20, 31, 10, 21};
- static const uint8_t SHIFT_LO_11[32] = {0, 0, 10, 0, 0, 9, 0, 0, 8, 0, 0, 7, 0, 0, 6, 0,
- 0, 5, 0, 0, 4, 0, 0, 3, 0, 0, 2, 0, 0, 1, 0, 0};
- static const uint8_t READ_FLAG_11[32] = {1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0,
- 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_11, SHIFT_HI_11,
- SHIFT_LO_11, READ_FLAG_11, mask11, nb11, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_11, SHIFT_HI_11, SHIFT_LO_11,
- READ_FLAG_11, mask11, nb11, isZigZag);
- }
-}
-
-static void bitunpack256v32_12_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb12 = 12;
- const uint32_t mask12 = (1u << nb12) - 1; // 0xFFF
- const int expansions_count_12 = 8;
- static const uint8_t SHIFT_HI_12[8] = {0, 12, 24, 4, 16, 28, 8, 20};
- static const uint8_t SHIFT_LO_12[8] = {0, 0, 8, 0, 0, 4, 0, 0};
- static const uint8_t READ_FLAG_12[8] = {1, 0, 1, 0, 0, 1, 0, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_12,
- SHIFT_HI_12, SHIFT_LO_12, READ_FLAG_12, mask12,
- nb12, isZigZag);
- }
- } else {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_12, SHIFT_HI_12, SHIFT_LO_12,
- READ_FLAG_12, mask12, nb12, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_13_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb13 = 13;
- const uint32_t mask13 = (1u << nb13) - 1; // 0x1FFF
- const int expansions_count_13 = 32;
- static const uint8_t SHIFT_HI_13[32] = {0, 13, 26, 7, 20, 1, 14, 27, 8, 21, 2,
- 15, 28, 9, 22, 3, 16, 29, 10, 23, 4, 17,
- 30, 11, 24, 5, 18, 31, 12, 25, 6, 19};
- static const uint8_t SHIFT_LO_13[32] = {0, 0, 6, 0, 12, 0, 0, 5, 0, 11, 0, 0, 4, 0, 10, 0,
- 0, 3, 0, 9, 0, 0, 2, 0, 8, 0, 0, 1, 0, 7, 0, 0};
- static const uint8_t READ_FLAG_13[32] = {1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0,
- 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_13, SHIFT_HI_13,
- SHIFT_LO_13, READ_FLAG_13, mask13, nb13, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_13, SHIFT_HI_13, SHIFT_LO_13,
- READ_FLAG_13, mask13, nb13, isZigZag);
- }
-}
-
-static void bitunpack256v32_14_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb14 = 14;
- const uint32_t mask14 = (1u << nb14) - 1; // 0x3FFF
- const int expansions_count_14 = 16;
- static const uint8_t SHIFT_HI_14[16] = {0, 14, 28, 10, 24, 6, 20, 2,
- 16, 30, 12, 26, 8, 22, 4, 18};
- static const uint8_t SHIFT_LO_14[16] = {0, 0, 4, 0, 8, 0, 12, 0, 0, 2, 0, 6, 0, 10, 0, 0};
- static const uint8_t READ_FLAG_14[16] = {1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_14,
- SHIFT_HI_14, SHIFT_LO_14, READ_FLAG_14, mask14,
- nb14, isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_14, SHIFT_HI_14, SHIFT_LO_14,
- READ_FLAG_14, mask14, nb14, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_15_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb15 = 15;
- const uint32_t mask15 = (1u << 15) - 1; // 0x7FFF
-
- // expansions=32 => unpacks 256 values at once
- const int expansions_count_15 = 32;
-
- static const uint8_t SHIFT_HI_15[32] = {0, 15, 30, 13, 28, 11, 26, 9, 24, 7, 22,
- 5, 20, 3, 18, 1, 16, 31, 14, 29, 12, 27,
- 10, 25, 8, 23, 6, 21, 4, 19, 2, 17};
-
- static const uint8_t SHIFT_LO_15[32] = {0, 0, 2, 0, 4, 0, 6, 0, 8, 0, 10, 0, 12, 0, 14, 0,
- 0, 1, 0, 3, 0, 5, 0, 7, 0, 9, 0, 11, 0, 13, 0, 0};
-
- static const uint8_t READ_FLAG_15[32] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
- 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_15, SHIFT_HI_15,
- SHIFT_LO_15, READ_FLAG_15, mask15, nb15, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_15, SHIFT_HI_15, SHIFT_LO_15,
- READ_FLAG_15, mask15, nb15, isZigZag);
- }
-}
-
-static void bitunpack256v32_16_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb16 = 16;
- const uint32_t mask16 = (1u << 16) - 1; // 0xFFFF
-
- const int expansions_count = 2;
- // Iteration 0: directly read 8×32-bit; Iteration 1: only right shift 16 bits, no new data read
- static const uint8_t SHIFT_HI_16[2] = {0, 16};
- static const uint8_t SHIFT_LO_16[2] = {0, 0};
- static const uint8_t READ_FLAG_16[2] = {1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 16; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count,
- SHIFT_HI_16, SHIFT_LO_16, READ_FLAG_16, mask16,
- nb16, isZigZag);
- }
- } else {
- for (int i = 0; i < 16; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI_16, SHIFT_LO_16,
- READ_FLAG_16, mask16, nb16, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_17_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const int nb17 = 17;
- const uint32_t mask17 = (1u << 17) - 1; // 0x1FFFF
-
- // expansions=32 => unpacks 256 values
- const int expansions_count_17 = 32;
-
- static const uint8_t SHIFT_HI_17[32] = {0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10,
- 27, 12, 29, 14, 31, 16, 1, 18, 3, 20, 5,
- 22, 7, 24, 9, 26, 11, 28, 13, 30, 15};
-
- static const uint8_t SHIFT_LO_17[32] = {0, 15, 0, 13, 0, 11, 0, 9, 0, 7, 0, 5, 0, 3, 0, 1,
- 16, 0, 14, 0, 12, 0, 10, 0, 8, 0, 6, 0, 4, 0, 2, 0};
-
- static const uint8_t READ_FLAG_17[32] = {1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
- 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_17, SHIFT_HI_17,
- SHIFT_LO_17, READ_FLAG_17, mask17, nb17, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_17, SHIFT_HI_17, SHIFT_LO_17,
- READ_FLAG_17, mask17, nb17, isZigZag);
- }
-}
-
-static void bitunpack256v32_18_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const int nb18 = 18;
- const uint32_t mask18 = (1u << 18) - 1; // 0x3FFFF
-
- // expansions=16 => 128 values
- const int expansions_count_18 = 16;
-
- static const uint8_t SHIFT_HI_18[16] = {0, 18, 4, 22, 8, 26, 12, 30,
- 16, 2, 20, 6, 24, 10, 28, 14};
- static const uint8_t SHIFT_LO_18[16] = {0, 14, 0, 10, 0, 6, 0, 2, 16, 0, 12, 0, 8, 0, 4, 0};
- static const uint8_t READ_FLAG_18[16] = {// #0 =>1, #1 =>1, #2=>0, #3=>1,
- // #4 =>0, #5 =>1, #6=>0, #7=>1,
- // #8 =>1, #9 =>0, #10=>1, #11=>0,
- // #12=>1, #13=>0, #14=>1, #15=>0
- 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_18,
- SHIFT_HI_18, SHIFT_LO_18, READ_FLAG_18, mask18,
- nb18, isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_18, SHIFT_HI_18, SHIFT_LO_18,
- READ_FLAG_18, mask18, nb18, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_19_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const int nb19 = 19;
- const uint32_t mask19 = (1u << 19) - 1; // 0x7FFFF
-
- // expansions=32 => unpacks 256 values at once
- const int expansions_count_19 = 32;
-
- static const uint8_t SHIFT_HI_19[32] = {0, 19, 6, 25, 12, 31, 18, 5, 24, 11, 30,
- 17, 4, 23, 10, 29, 16, 3, 22, 9, 28, 15,
- 2, 21, 8, 27, 14, 1, 20, 7, 26, 13};
- static const uint8_t SHIFT_LO_19[32] = {0, 13, 0, 7, 0, 1, 14, 0, 8, 0, 2, 15, 0, 9, 0, 3,
- 16, 0, 10, 0, 4, 17, 0, 11, 0, 5, 18, 0, 12, 0, 6, 0};
- static const uint8_t READ_FLAG_19[32] = {1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1,
- 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_19, SHIFT_HI_19,
- SHIFT_LO_19, READ_FLAG_19, mask19, nb19, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_19, SHIFT_HI_19, SHIFT_LO_19,
- READ_FLAG_19, mask19, nb19, isZigZag);
- }
-}
-
-static void bitunpack256v32_20_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const int nb20 = 20;
- const uint32_t mask20 = (1u << 20) - 1; // 0xFFFFF
-
- // expansions=8 => process 64 values at once
- const int expansions_count_20 = 8;
-
- // shift tables for k=0..7
- static const uint8_t SHIFT_HI_20[8] = {0, 20, 8, 28, 16, 4, 24, 12};
- static const uint8_t SHIFT_LO_20[8] = {0, 12, 0, 4, 16, 0, 8, 0};
- static const uint8_t READ_FLAG_20[8] = {1, 1, 0, 1, 1, 0, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_20,
- SHIFT_HI_20, SHIFT_LO_20, READ_FLAG_20, mask20,
- nb20, isZigZag);
- }
- } else {
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_20, SHIFT_HI_20, SHIFT_LO_20,
- READ_FLAG_20, mask20, nb20, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_21_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const uint32_t mask21 = (1u << 21) - 1; // 0x1FFFFF
- const int nb21 = 21;
-
- // expansions=32 => unpacks 256 values at once
- const int expansions_count_21 = 32;
-
- static const uint8_t SHIFT_HI_21[32] = {0, 21, 10, 31, 20, 9, 30, 19, 8, 29, 18,
- 7, 28, 17, 6, 27, 16, 5, 26, 15, 4, 25,
- 14, 3, 24, 13, 2, 23, 12, 1, 22, 11};
- static const uint8_t SHIFT_LO_21[32] = {0, 11, 0, 1, 12, 0, 2, 13, 0, 3, 14,
- 0, 4, 15, 0, 5, 16, 0, 6, 17, 0, 7,
- 18, 0, 8, 19, 0, 9, 20, 0, 10, 0};
- static const uint8_t READ_FLAG_21[32] = {
- // Check original expansions #k if there's a "load #X" => 1 if yes, 0 if no
- 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
- 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_21, SHIFT_HI_21,
- SHIFT_LO_21, READ_FLAG_21, mask21, nb21, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_21, SHIFT_HI_21, SHIFT_LO_21,
- READ_FLAG_21, mask21, nb21, isZigZag);
- }
-}
-
-static void bitunpack256v32_22_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const uint32_t mask22 = (1u << 22) - 1; // 0x3FFFFF
- const int nb22 = 22;
-
- // b=22 => one block function with expansions=16 => outputs 128 values
- // need to call it twice to get 256 values
- const int expansions_count_22 = 16;
-
- static const uint8_t SHIFT_HI_22[16] = {/* 0 */ 0, /* 1 */ 22, /* 2 */ 12, /* 3 */ 2,
- /* 4 */ 24, /* 5 */ 14, /* 6 */ 4, /* 7 */ 26,
- /* 8 */ 16, /* 9 */ 6, /*10 */ 28, /*11 */ 18,
- /*12 */ 8, /*13 */ 30, /*14 */ 20, /*15 */ 10};
-
- static const uint8_t SHIFT_LO_22[16] = {/* 0 */ 0, /* 1 */ 10, /* 2 */ 20, /* 3 */ 0,
- /* 4 */ 8, /* 5 */ 18, /* 6 */ 0, /* 7 */ 6,
- /* 8 */ 16, /* 9 */ 0, /*10 */ 4, /*11 */ 14,
- /*12 */ 0, /*13 */ 2, /*14 */ 12, /*15 */ 0};
-
- static const uint8_t READ_FLAG_22[16] = {
- // From original code: expansions #3, #6, #9, #12, #15 don't read, others do
- 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_22,
- SHIFT_HI_22, SHIFT_LO_22, READ_FLAG_22, mask22,
- nb22, isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_22, SHIFT_HI_22, SHIFT_LO_22,
- READ_FLAG_22, mask22, nb22, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_23_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const int nb23 = 23;
- const uint32_t mask23 = (1u << 23) - 1; // 0x7FFFFF
-
- // expansions_count=32
- const int expansions_count_23 = 32;
-
- // Predefined SHIFT_HI_23, SHIFT_LO_23, READ_FLAG_23
- static const uint8_t SHIFT_HI_23[32] = {0, 23, 14, 5, 28, 19, 10, 1, 24, 15, 6,
- 29, 20, 11, 2, 25, 16, 7, 30, 21, 12, 3,
- 26, 17, 8, 31, 22, 13, 4, 27, 18, 9};
-
- static const uint8_t SHIFT_LO_23[32] = {0, 9, 18, 0, 4, 13, 22, 0, 8, 17, 0,
- 3, 12, 21, 0, 7, 16, 0, 2, 11, 20, 0,
- 6, 15, 0, 1, 10, 19, 0, 5, 14, 0};
-
- static const uint8_t READ_FLAG_23[32] = {1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1,
- 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_23, SHIFT_HI_23,
- SHIFT_LO_23, READ_FLAG_23, mask23, nb23, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_23, SHIFT_HI_23, SHIFT_LO_23,
- READ_FLAG_23, mask23, nb23, isZigZag);
- }
-}
-static void bitunpack256v32_24_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // base bits & mask
- const int nb24 = 24;
- const uint32_t mask24 = (1u << 24) - 1; // 0xFFFFFF
-
- // expansions_count=4 (corresponds to 4 expansions => outputs 32 values)
- const int expansions_count_24 = 4;
-
- // k=0 => leftover>>0, new<<0
- // k=1 => leftover>>24, new<<8
- // k=2 => leftover>>16, new<<16
- // k=3 => leftover>>8, no new block read
- static const uint8_t SHIFT_HI_24[4] = {0, 24, 16, 8};
- static const uint8_t SHIFT_LO_24[4] = {0, 8, 16, 0};
-
- // Only read new blocks for steps 0,1,2, not for step 3
- static const uint8_t READ_FLAG_24[4] = {1, 1, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 8; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_24,
- SHIFT_HI_24, SHIFT_LO_24, READ_FLAG_24, mask24,
- nb24, isZigZag);
- }
- } else {
- for (int i = 0; i < 8; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_24, SHIFT_HI_24, SHIFT_LO_24,
- READ_FLAG_24, mask24, nb24, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_25_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // mask & base bits
- const uint32_t mask25 = (1u << 25) - 1; // 0x1FFFFFF
- const int nb25 = 25;
-
- // 32 expansions total
- const int expansions_count_25 = 32;
-
- // Extract high and low shift amounts from original implementation
- static const uint8_t SHIFT_HI_25[32] = {
- /* #0 */ 0, /* #1 */ 25, /* #2 */ 18, /* #3 */ 11,
- /* #4 */ 4, /* #5 */ 29, /* #6 */ 22, /* #7 */ 15,
- /* #8 */ 8, /* #9 */ 1, /* #10 */ 26, /* #11 */ 19,
- /* #12 */ 12, /* #13 */ 5, /* #14 */ 30, /* #15 */ 23,
- /* #16 */ 16, /* #17 */ 9, /* #18 */ 2, /* #19 */ 27,
- /* #20 */ 20, /* #21 */ 13, /* #22 */ 6, /* #23 */ 31,
- /* #24 */ 24, /* #25 */ 17, /* #26 */ 10, /* #27 */ 3,
- /* #28 */ 28, /* #29 */ 21, /* #30 */ 14, /* #31 */ 7};
-
- static const uint8_t SHIFT_LO_25[32] = {
- /* #0 */ 0, /* #1 */ 7, /* #2 */ 14, /* #3 */ 21,
- /* #4 */ 0, /* #5 */ 3, /* #6 */ 10, /* #7 */ 17,
- /* #8 */ 24, /* #9 */ 0, /* #10 */ 6, /* #11 */ 13,
- /* #12 */ 20, /* #13 */ 0, /* #14 */ 2, /* #15 */ 9,
- /* #16 */ 16, /* #17 */ 23, /* #18 */ 0, /* #19 */ 5,
- /* #20 */ 12, /* #21 */ 19, /* #22 */ 0, /* #23 */ 1,
- /* #24 */ 8, /* #25 */ 15, /* #26 */ 22, /* #27 */ 0,
- /* #28 */ 4, /* #29 */ 11, /* #30 */ 18, /* #31 */ 0};
-
- // Mark which steps don't need to read new data
- // Based on original code, expansions #4, #9, #13, #18, #22, #27, #31 don't need to read new data
- static const uint8_t READ_FLAG_25[32] = {
- /* #0 */ 1, /* #1 */ 1, /* #2 */ 1, /* #3 */ 1,
- /* #4 */ 0, /* #5 */ 1, /* #6 */ 1, /* #7 */ 1,
- /* #8 */ 1, /* #9 */ 0, /* #10 */ 1, /* #11 */ 1,
- /* #12 */ 1, /* #13 */ 0, /* #14 */ 1, /* #15 */ 1,
- /* #16 */ 1, /* #17 */ 1, /* #18 */ 0, /* #19 */ 1,
- /* #20 */ 1, /* #21 */ 1, /* #22 */ 0, /* #23 */ 1,
- /* #24 */ 1, /* #25 */ 1, /* #26 */ 1, /* #27 */ 0,
- /* #28 */ 1, /* #29 */ 1, /* #30 */ 1, /* #31 */ 0};
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_25,
- SHIFT_HI_25, SHIFT_LO_25, READ_FLAG_25, mask25,
- nb25, isZigZag);
- }
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_25, SHIFT_HI_25, SHIFT_LO_25,
- READ_FLAG_25, mask25, nb25, isZigZag);
- }
-}
-
-static void bitunpack256v32_26_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // mask & base bits
- const uint32_t mask26 = (1u << 26) - 1; // 0x3FFFFFF
- const int nb26 = 26;
-
- // 16 expansions total
- const int expansions_count_26 = 16;
-
- // Extract high and low shift amounts from original implementation
- static const uint8_t SHIFT_HI_26[16] = {
- /* #0 */ 0, /* #1 */ 26, /* #2 */ 20, /* #3 */ 14,
- /* #4 */ 8, /* #5 */ 2, /* #6 */ 28, /* #7 */ 22,
- /* #8 */ 16, /* #9 */ 10, /* #10 */ 4, /* #11 */ 30,
- /* #12 */ 24, /* #13 */ 18, /* #14 */ 12, /* #15 */ 6};
-
- static const uint8_t SHIFT_LO_26[16] = {
- /* #0 */ 0, /* #1 */ 6, /* #2 */ 12, /* #3 */ 18,
- /* #4 */ 24, /* #5 */ 0, /* #6 */ 4, /* #7 */ 10,
- /* #8 */ 16, /* #9 */ 22, /* #10 */ 0, /* #11 */ 2,
- /* #12 */ 8, /* #13 */ 14, /* #14 */ 20, /* #15 */ 0};
-
- // Mark which steps don't need to read new data
- // Based on original code, expansions #5, #10, #15 don't need to read new data
- static const uint8_t READ_FLAG_26[16] = {
- /* #0 */ 1, /* #1 */ 1, /* #2 */ 1, /* #3 */ 1,
- /* #4 */ 1, /* #5 */ 0, /* #6 */ 1, /* #7 */ 1,
- /* #8 */ 1, /* #9 */ 1, /* #10 */ 0, /* #11 */ 1,
- /* #12 */ 1, /* #13 */ 1, /* #14 */ 1, /* #15 */ 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_26,
- SHIFT_HI_26, SHIFT_LO_26, READ_FLAG_26, mask26,
- nb26, isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_26, SHIFT_HI_26, SHIFT_LO_26,
- READ_FLAG_26, mask26, nb26, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_27_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- // mask & base bits
- const uint32_t mask27 = (1u << 27) - 1; // 0x7FFFFFF
- const int nb27 = 27;
-
- // 32 expansions total
- const int expansions_count_27 = 32;
-
- // Extract high and low shift amounts from original implementation
- static const uint8_t SHIFT_HI_27[32] = {
- /* #0 */ 0, /* #1 */ 27, /* #2 */ 22, /* #3 */ 17,
- /* #4 */ 12, /* #5 */ 7, /* #6 */ 2, /* #7 */ 29,
- /* #8 */ 24, /* #9 */ 19, /* #10 */ 14, /* #11 */ 9,
- /* #12 */ 4, /* #13 */ 31, /* #14 */ 26, /* #15 */ 21,
- /* #16 */ 16, /* #17 */ 11, /* #18 */ 6, /* #19 */ 1,
- /* #20 */ 28, /* #21 */ 23, /* #22 */ 18, /* #23 */ 13,
- /* #24 */ 8, /* #25 */ 3, /* #26 */ 30, /* #27 */ 25,
- /* #28 */ 20, /* #29 */ 15, /* #30 */ 10, /* #31 */ 5};
-
- static const uint8_t SHIFT_LO_27[32] = {
- /* #0 */ 0, /* #1 */ 5, /* #2 */ 10, /* #3 */ 15,
- /* #4 */ 20, /* #5 */ 25, /* #6 */ 0, /* #7 */ 3,
- /* #8 */ 8, /* #9 */ 13, /* #10 */ 18, /* #11 */ 23,
- /* #12 */ 0, /* #13 */ 1, /* #14 */ 6, /* #15 */ 11,
- /* #16 */ 16, /* #17 */ 21, /* #18 */ 26, /* #19 */ 0,
- /* #20 */ 4, /* #21 */ 9, /* #22 */ 14, /* #23 */ 19,
- /* #24 */ 24, /* #25 */ 0, /* #26 */ 2, /* #27 */ 7,
- /* #28 */ 12, /* #29 */ 17, /* #30 */ 22, /* #31 */ 0};
-
- // Mark which steps don't need to read new data
- // From original code, steps #6, #12, #19, #25, #31 don't have CPY8(iv, *pIn)
- static const uint8_t READ_FLAG_27[32] = {
- /* #0 */ 1, /* #1 */ 1, /* #2 */ 1, /* #3 */ 1,
- /* #4 */ 1, /* #5 */ 1, /* #6 */ 0, /* #7 */ 1,
- /* #8 */ 1, /* #9 */ 1, /* #10 */ 1, /* #11 */ 1,
- /* #12 */ 0, /* #13 */ 1, /* #14 */ 1, /* #15 */ 1,
- /* #16 */ 1, /* #17 */ 1, /* #18 */ 1, /* #19 */ 0,
- /* #20 */ 1, /* #21 */ 1, /* #22 */ 1, /* #23 */ 1,
- /* #24 */ 1, /* #25 */ 0, /* #26 */ 1, /* #27 */ 1,
- /* #28 */ 1, /* #29 */ 1, /* #30 */ 1, /* #31 */ 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_27, SHIFT_HI_27,
- SHIFT_LO_27, READ_FLAG_27, mask27, nb27, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_27, SHIFT_HI_27, SHIFT_LO_27,
- READ_FLAG_27, mask27, nb27, isZigZag);
- }
-}
-static void bitunpack256v32_28_scalar(
- uint32_t** pIn, uint32_t** pOut,
- uint32_t** pPEX, // Optional parameter, non-NULL for extended version
- unsigned char** pBB, bool isZigZag) // Optional parameter, non-NULL for extended version
-{
- // Common constant definitions
- const uint32_t mask28 = (1u << 28) - 1; // 0xFFFFFFF
- const int nb28 = 28;
- const int expansions_count_28 = 8;
- static const uint8_t SHIFT_HI_28[8] = {0, 28, 24, 20, 16, 12, 8, 4};
- static const uint8_t SHIFT_LO_28[8] = {0, 4, 8, 12, 16, 20, 24, 0};
- static const uint8_t READ_FLAG_28[8] = {1, 1, 1, 1, 1, 1, 1, 0};
-
- // Choose template based on whether extension parameters are provided
- if (pPEX != NULL && pBB != NULL) {
- // Call extended template, each call outputs 64 values, loop 4 times to get 256
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count_28,
- SHIFT_HI_28, SHIFT_LO_28, READ_FLAG_28, mask28,
- nb28, isZigZag);
- }
- } else {
- // Call non-extended template, also each call outputs 64 values, loop 4 times to get 256
- for (int i = 0; i < 4; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count_28, SHIFT_HI_28, SHIFT_LO_28,
- READ_FLAG_28, mask28, nb28, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_29_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const uint32_t mask29 = (1U << 29) - 1; // 0x1FFFFFFF
- const int expansions_count = 32;
- static const uint8_t SHIFT_HI_29[32] = {0, 29, 26, 23, 20, 17, 14, 11, 8, 5, 2,
- 31, 28, 25, 22, 19, 16, 13, 10, 7, 4, 1,
- 30, 27, 24, 21, 18, 15, 12, 9, 6, 3};
- static const uint8_t SHIFT_LO_29[32] = {0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 0,
- 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 0,
- 2, 5, 8, 11, 14, 17, 20, 23, 26, 0};
- static const uint8_t READ_FLAG_29[32] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count, SHIFT_HI_29,
- SHIFT_LO_29, READ_FLAG_29, mask29, 29, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI_29, SHIFT_LO_29,
- READ_FLAG_29, mask29, 29, isZigZag);
- }
-}
-
-static void bitunpack256v32_30_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const uint32_t mask30 = (1U << 30) - 1; // 0x3FFFFFFF
- const int expansions_count = 16;
- static const uint8_t SHIFT_HI_30[16] = {0, 30, 28, 26, 24, 22, 20, 18,
- 16, 14, 12, 10, 8, 6, 4, 2};
- static const uint8_t SHIFT_LO_30[16] = {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 0};
- static const uint8_t READ_FLAG_30[16] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0};
-
- if (pPEX != NULL && pBB != NULL) {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count,
- SHIFT_HI_30, SHIFT_LO_30, READ_FLAG_30, mask30, 30,
- isZigZag);
- }
- } else {
- for (int i = 0; i < 2; i++) {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI_30, SHIFT_LO_30,
- READ_FLAG_30, mask30, 30, isZigZag);
- }
- }
-}
-
-static void bitunpack256v32_31_scalar(uint32_t** pIn, uint32_t** pOut, uint32_t** pPEX,
- unsigned char** pBB, bool isZigZag) {
- const uint32_t mask31 = (1U << 31) - 1; // 0x7FFFFFFF
- const int expansions_count = 32;
- // Construct parameter arrays:
- // For k==0: SHIFT_HI = 0, SHIFT_LO = 0, READ_FLAG = 1
- // For k = 1 .. 30: SHIFT_HI = 32 - k, SHIFT_LO = k, READ_FLAG = 1
- // For k==31: SHIFT_HI = 1, SHIFT_LO = 0, READ_FLAG = 0
- static const uint8_t SHIFT_HI[32] = {0, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22,
- 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
- 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
- static const uint8_t SHIFT_LO[32] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
- 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
- 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0};
- static const uint8_t READ_FLAG[32] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0};
- if (pPEX != NULL && pBB != NULL) {
- bitunblk256v32_scalarBlock_ex_template(pIn, pOut, pPEX, pBB, expansions_count, SHIFT_HI,
- SHIFT_LO, READ_FLAG, mask31, 31, isZigZag);
- } else {
- bitunblk256v32_scalar_template(pIn, pOut, expansions_count, SHIFT_HI, SHIFT_LO, READ_FLAG,
- mask31, 31, isZigZag);
- }
-}
-
-static void bitunpack256v32_32_scalar(
- uint32_t** pIn, uint32_t** pOut,
- uint32_t** pPEX, // Optional parameter, non-NULL for extended version
- unsigned char** pBB, bool isZigZag) // Optional parameter, non-NULL for extended version
-{
- uint32_t* ip = *pIn;
- uint32_t* op = *pOut;
- const int nb = 32; // When b=32, each 32-bit integer stores a value directly
-
- // There are 32 groups, each group has 8 numbers, totaling 256 numbers
- for (int i = 0; i < 32; i++) {
- // Copy 8 input values directly to output (avoid calling CPY8)
- for (int j = 0; j < 8; j++) {
... 18722 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org