You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ai...@apache.org on 2026/10/23 09:54:42 UTC
(doris-thirdparty) branch clucene-2.1 updated: [refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#424)
This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch clucene-2.1
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene-2.1 by this push:
new d6081efc40b [refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#424)
d6081efc40b is described below
commit d6081efc40b865defd019d93835c4651e32b43bb
Author: Jack <ji...@selectdb.com>
AuthorDate: Fri Oct 23 17:54:36 2026 +0800
[refactor](clucene) Replace the GPL TurboPFor library with an independent PFOR codec (#421) (#424)
### What problem does this PR solve?
Issue Number: None
Related PR: #421
Problem Summary:
Pick of #421 to clucene-2.1. `src/ext/for` is a vendored copy of powturbo's TurboPFor, which is licensed under GPL v2, linked into clucene-core and thereby into every Doris BE. It is replaced by `CLucene/util/PForCodec.{h,cpp}`, an independent implementation of the same on-disk block format (bit compatible in both directions), and `src/ext/for`, the `ic` library target and the `ic` link dependencies are removed. See #421 for the details, the measurements and the tests of the codec.
Differences to the original change: this branch only has the CodeMode `kPfor`, no `PforUtil::pfor_encode/pfor_decode`, and the P4ENC / P4DEC / P4NZENC / P4NZDEC entry points are used directly by the readers and writers. Those keep their API and their choice of the format by platform, only the TurboPFor functions behind them are replaced; `PforUtil::decodePos` calls the bounded decoder. The branch had no testPFOR, so it gets one with the codec tests (golden vectors from the previous en [...]
### Release note
Removed the GPL v2 licensed TurboPFor library from the inverted index code. The index format is unchanged.
### Check List (For Author)
- Test: Unit Test
- Full cl_test (62 tests, 5 of them new) built with clang + AVX2 (with `-include memory`, see above) and with g++-11 and the default options: all passed.
- Behavior changed: No
- Does this need documentation: No
(cherry picked from commit 5725b460b6b12a6874b171832f9f834ce161b2f4)
Co-authored-by: Claude Sonnet 5.5 <no...@anthropic.com>
---
CMakeLists.txt | 2 -
cmake/TurboPFOR.cmake | 15 -
src/core/CLucene/util/PFORUtil.cpp | 62 +-
src/core/CLucene/util/PForCodec.cpp | 846 ++++
src/core/CLucene/util/PForCodec.h | 93 +
src/core/CMakeLists.txt | 2 +-
src/demo/CMakeLists.txt | 20 +-
src/ext/for/CMakeLists.txt | 91 -
src/ext/for/README.md | 585 ---
src/ext/for/bitpack.c | 427 --
src/ext/for/bitpack.h | 310 --
src/ext/for/bitpack_.h | 5190 -------------------
src/ext/for/bitunpack.c | 1231 -----
src/ext/for/bitunpack_.h | 6049 -----------------------
src/ext/for/bitutil.c | 689 ---
src/ext/for/bitutil.h | 547 --
src/ext/for/conf.h | 282 --
src/ext/for/fp.h | 125 -
src/ext/for/makefile | 156 -
src/ext/for/makefile.vs | 78 -
src/ext/for/sse_neon.h | 355 --
src/ext/for/time_.h | 252 -
src/ext/for/transpose.c | 1223 -----
src/ext/for/transpose.h | 113 -
src/ext/for/trle.h | 72 -
src/ext/for/trle_.h | 60 -
src/ext/for/trlec.c | 344 --
src/ext/for/trled.c | 421 --
src/ext/for/v8.c | 1466 ------
src/ext/for/vint.c | 407 --
src/ext/for/vint.h | 401 --
src/ext/for/vp4.h | 355 --
src/ext/for/vp4c.c | 423 --
src/ext/for/vp4d.c | 534 --
src/ext/for/vs/bitpack_avx2.c | 2 -
src/ext/for/vs/bitpack_sse.c | 2 -
src/ext/for/vs/bitunpack_avx2.c | 2 -
src/ext/for/vs/bitunpack_sse.c | 2 -
src/ext/for/vs/getopt.c | 562 ---
src/ext/for/vs/getopt.h | 97 -
src/ext/for/vs/inttypes.h | 306 --
src/ext/for/vs/stdint.h | 259 -
src/ext/for/vs/transpose_avx2.c | 2 -
src/ext/for/vs/transpose_sse.c | 2 -
src/ext/for/vs/vp4c_avx2.c | 2 -
src/ext/for/vs/vp4c_sse.c | 2 -
src/ext/for/vs/vp4d_avx2.c | 2 -
src/ext/for/vs/vp4d_sse.c | 2 -
src/ext/for/vs/vs2017/TurboPFor.sln | 41 -
src/ext/for/vs/vs2017/TurboPFor.vcxproj | 226 -
src/ext/for/vs/vs2017/TurboPFor.vcxproj.filters | 101 -
src/ext/for/vs/vs2017/icapp.vcxproj | 175 -
src/ext/for/vs/vs2017/icapp.vcxproj.filters | 21 -
src/ext/for/vsimple.c | 537 --
src/ext/for/vsimple.h | 47 -
src/test/CMakeLists.txt | 11 +-
src/test/data/pfor_codec_golden.dat | Bin 0 -> 106756 bytes
src/test/store/testPFOR.cpp | 266 +
src/test/test.h | 1 +
src/test/tests.cpp | 1 +
60 files changed, 1272 insertions(+), 24625 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 655e0e8784c..d2ba2defb6f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -195,8 +195,6 @@ ADD_SUBDIRECTORY (src/ext)
include(cmake/FindRoaring.cmake)
find_package(Roaring REQUIRED)
INCLUDE_DIRECTORIES(${clucene_SOURCE_DIR}/src/ext/sse2neon)
-#TurboPFOR
-ADD_SUBDIRECTORY (src/ext/for)
ADD_SUBDIRECTORY (src/shared)
ADD_SUBDIRECTORY (src/core)
ADD_SUBDIRECTORY (src/test)
diff --git a/cmake/TurboPFOR.cmake b/cmake/TurboPFOR.cmake
deleted file mode 100644
index a742968728c..00000000000
--- a/cmake/TurboPFOR.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-PROJECT(turbo-pfor)
-set(PFOR_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/ext/for)
-
-add_custom_target(build_ic ALL
- COMMAND make OPT='-Wno-implicit-function-declaration -Wno-implicit-int -Wno-int-conversion' USE_AVX2=${USE_AVX2} libic.a -j 8
- WORKING_DIRECTORY ${PFOR_SOURCE_DIR}
- COMMENT "Original Turbo-PFOR makefile target")
-
-add_library(ic STATIC IMPORTED)
-set_target_properties(ic PROPERTIES IMPORTED_LOCATION "${PFOR_SOURCE_DIR}/libic.a")
-add_dependencies(ic build_ic)
-
-install(FILES ${PFOR_SOURCE_DIR}/libic.a
- DESTINATION "lib"
- COMPONENT development)
diff --git a/src/core/CLucene/util/PFORUtil.cpp b/src/core/CLucene/util/PFORUtil.cpp
index 8ebad306a4e..4d3f35e4e4c 100644
--- a/src/core/CLucene/util/PFORUtil.cpp
+++ b/src/core/CLucene/util/PFORUtil.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "PFORUtil.h"
-#include "vp4.h"
+#include "CLucene/util/PForCodec.h"
#if (defined(__i386) || defined(__x86_64__))
#include <cpuid.h>
#endif
@@ -26,6 +26,37 @@
CL_NS_USE(index)
namespace {
+// The P4* entry points take no length. They read from buffers that the callers pad by
+// PFOR_BLOCK_SIZE, which is what the decoders need after the end of a stream.
+size_t decodeDelta1_256(unsigned char* in, size_t n, uint32_t* out) {
+ return lucene::util::pfor::decodeDelta1_256(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+}
+size_t decodeZigzag256(unsigned char* in, size_t n, uint32_t* out) {
+ return lucene::util::pfor::decodeZigzag256(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+}
+size_t decodeDelta1_128(unsigned char* in, size_t n, uint32_t* out) {
+ return lucene::util::pfor::decodeDelta1_128(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+}
+size_t decodeZigzag128(unsigned char* in, size_t n, uint32_t* out) {
+ return lucene::util::pfor::decodeZigzag128(in, in + lucene::util::pfor::encodedSizeBound(n), n,
+ out);
+}
+size_t encodeDelta1_256(uint32_t* in, size_t n, unsigned char* out) {
+ return lucene::util::pfor::encodeDelta1_256(in, n, out);
+}
+size_t encodeZigzag256(uint32_t* in, size_t n, unsigned char* out) {
+ return lucene::util::pfor::encodeZigzag256(in, n, out);
+}
+size_t encodeDelta1_128(uint32_t* in, size_t n, unsigned char* out) {
+ return lucene::util::pfor::encodeDelta1_128(in, n, out);
+}
+size_t encodeZigzag128(uint32_t* in, size_t n, unsigned char* out) {
+ return lucene::util::pfor::encodeZigzag128(in, n, out);
+}
+
using DEC_FUNC = size_t (*)(unsigned char *__restrict, size_t, uint32_t *__restrict);
using ENC_FUNC = size_t (*)(uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
DEC_FUNC g_p4nd1dec;
@@ -101,10 +132,10 @@ __attribute__((constructor)) void SelectPFORFunctions() {
bool sse2 = (edx & bit_SSE2) != 0;
bool sse42 = (ecx & bit_SSE4_2) != 0;
#if defined(USE_AVX2)
- g_p4nd1dec = p4nd1dec256v32;
- g_p4nzdec = p4nzdec256v32;
- g_p4nd1enc = p4nd1enc256v32;
- g_p4nzenc = p4nzenc256v32;
+ g_p4nd1dec = decodeDelta1_256;
+ g_p4nzdec = decodeZigzag256;
+ g_p4nd1enc = encodeDelta1_256;
+ g_p4nzenc = encodeZigzag256;
#else
g_p4nd1dec = DefaultDDEC;
g_p4nzdec = DefaultDEC;
@@ -112,10 +143,10 @@ __attribute__((constructor)) void SelectPFORFunctions() {
g_p4nzenc = DefaultENC;
#endif
#else
- g_p4nd1dec = p4nd1dec32;
- g_p4nzdec = p4nzdec32;
- g_p4nd1enc = p4nd1enc32;
- g_p4nzenc = p4nzenc32;
+ g_p4nd1dec = decodeDelta1_128;
+ g_p4nzdec = decodeZigzag128;
+ g_p4nd1enc = encodeDelta1_128;
+ g_p4nzenc = encodeZigzag128;
#endif
}
@@ -174,7 +205,18 @@ uint32_t PforUtil::decodePos(IndexInput* in, std::vector<uint32_t>& buffer) {
uint32_t serializedSize = in->readVInt();
std::vector<uint8_t> buf(serializedSize + PFOR_BLOCK_SIZE);
in->readBytes(buf.data(), serializedSize);
- P4NZDEC(buf.data(), size, buffer.data());
+ // the same function as P4NZDEC, with the length of the stream
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ lucene::util::pfor::decodeZigzag256(buf.data(), buf.data() + serializedSize, size,
+ buffer.data());
+#else
+ DefaultDEC(buf.data(), size, buffer.data());
+#endif
+#else
+ lucene::util::pfor::decodeZigzag128(buf.data(), buf.data() + serializedSize, size,
+ buffer.data());
+#endif
} else {
for (uint32_t i = 0; i < size; i++) {
buffer[i] = in->readVInt();
diff --git a/src/core/CLucene/util/PForCodec.cpp b/src/core/CLucene/util/PForCodec.cpp
new file mode 100644
index 00000000000..bbb1fae031a
--- /dev/null
+++ b/src/core/CLucene/util/PForCodec.cpp
@@ -0,0 +1,846 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "PForCodec.h"
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <utility>
+
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
+
+#include "CLucene/_ApiHeader.h"
+#include "CLucene/debug/error.h"
+
+static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "the stream layout assumes little endian");
+
+namespace lucene {
+namespace util {
+namespace pfor {
+
+namespace {
+
+constexpr uint32_t kModePlain = 0;
+constexpr uint32_t kModeList = 1;
+constexpr uint32_t kModeBitmap = 2;
+constexpr uint32_t kModeConst = 3;
+
+constexpr size_t kGroup = 32; // values per packing unit
+constexpr size_t kLanes = 8; // lanes of the vertical layout
+
+#if defined(__clang__)
+#define PFOR_UNROLL _Pragma("clang loop unroll(full)")
+#elif defined(__GNUC__)
+#define PFOR_UNROLL _Pragma("GCC unroll 32")
+#else
+#define PFOR_UNROLL
+#endif
+
+[[noreturn]] void corrupt(const char* what) {
+ _CLTHROWA(CL_ERR_CorruptIndex, what);
+}
+
+inline uint32_t bitLength(uint32_t x) {
+ return x == 0 ? 0 : 32 - static_cast<uint32_t>(__builtin_clz(x));
+}
+
+inline uint32_t load32(const uint8_t* p, size_t bytes) {
+ uint32_t v = 0;
+ memcpy(&v, p, bytes);
+ return v;
+}
+
+// ---------------------------------------------------------------------------------
+// Delta transforms
+// ---------------------------------------------------------------------------------
+
+// Eight 32-bit lanes; the compiler maps them to whatever vector registers the target has. The
+// functions passing them by value are internal, so the ABI note of GCC does not matter.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic ignored "-Wpsabi"
+#endif
+typedef uint32_t U32x8 __attribute__((vector_size(32)));
+typedef uint32_t U32x8Unaligned __attribute__((vector_size(32), aligned(4), may_alias));
+
+// Shuffles the lanes of two vectors, indices 0-7 pick from a and 8-15 from b. Clang and GCC 12+
+// know __builtin_shufflevector, older GCC only has __builtin_shuffle.
+#if defined(__clang__)
+#define PFOR_SHUFFLE(a, b, ...) __builtin_shufflevector(a, b, __VA_ARGS__)
+#else
+#define PFOR_SHUFFLE(a, b, ...) __builtin_shuffle(a, b, (U32x8) {__VA_ARGS__})
+#endif
+
+struct ZigzagDelta {
+ static uint32_t forward(uint32_t cur, uint32_t prev) {
+ uint32_t d = cur - prev;
+ return (d << 1) ^ (0u - (d >> 31));
+ }
+ // The signed difference a transformed value stands for.
+ static uint32_t toDelta(uint32_t t) { return (t >> 1) ^ (0u - (t & 1)); }
+ static U32x8 toDelta(U32x8 t) { return (t >> 1) ^ (0u - (t & 1u)); }
+};
+
+struct Delta1 {
+ static uint32_t forward(uint32_t cur, uint32_t prev) { return cur - prev - 1; }
+ static uint32_t toDelta(uint32_t t) { return t + 1; }
+ static U32x8 toDelta(U32x8 t) { return t + 1u; }
+};
+
+// ---------------------------------------------------------------------------------
+// Bit unpacking. A unit is 32 rows x L lanes of b-bit values stored in b*L words;
+// word j of lane l is at j*L+l and rows are packed LSB first. With L == 1 this is the
+// plain horizontal packing of 32 consecutive values, with L == 8 the vertical layout.
+// ---------------------------------------------------------------------------------
+
+#ifdef __AVX2__
+// Horizontal unpacking with AVX2. Eight consecutive b-bit values occupy exactly b bytes, so the
+// pattern of a group of eight is the same for the four groups of a unit. For b <= 25 every
+// value lies within four bytes: the bytes of the first four values are picked from 16 bytes
+// loaded at the start of the group, those of the other four from 16 bytes loaded at byte
+// 4*b/8, then shifted right by their bit offset.
+constexpr int kMaxShuffledWidth = 25;
+// The loads of the last group reach this many bytes past the 4*b bytes of a unit.
+constexpr size_t kUnpackOverread = 16;
+
+struct HorizontalTables {
+ alignas(32) uint8_t shuffle[kMaxShuffledWidth + 1][32];
+ alignas(32) uint32_t shift[kMaxShuffledWidth + 1][8];
+};
+
+constexpr HorizontalTables makeHorizontalTables() {
+ HorizontalTables t = {};
+ for (int b = 1; b <= kMaxShuffledWidth; ++b) {
+ for (int k = 0; k < 8; ++k) {
+ const int half = k / 4;
+ const int rel = k * b - (half ? 8 * ((4 * b) >> 3) : 0);
+ t.shift[b][k] = rel & 7;
+ for (int j = 0; j < 4; ++j) {
+ t.shuffle[b][half * 16 + (k % 4) * 4 + j] = (rel >> 3) + j;
+ }
+ }
+ }
+ return t;
+}
+
+constexpr HorizontalTables kHorizontalTables = makeHorizontalTables();
+#else
+constexpr size_t kUnpackOverread = 0;
+#endif
+
+#ifdef __AVX2__
+// Eight consecutive b-bit values, which occupy the b bytes at p (plus the overread of the loads).
+template <int B>
+inline void unpackGroup8(const uint8_t* p, uint32_t* out) {
+ if constexpr (B >= 1 && B <= kMaxShuffledWidth) {
+ const __m256i shuffle =
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kHorizontalTables.shuffle[B]));
+ const __m256i shift =
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kHorizontalTables.shift[B]));
+ const __m256i mask = _mm256_set1_epi32(static_cast<int>((1u << B) - 1));
+ const __m256i bytes = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))),
+ _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + ((4 * B) >> 3))), 1);
+ _mm256_storeu_si256(
+ reinterpret_cast<__m256i*>(out),
+ _mm256_and_si256(_mm256_srlv_epi32(_mm256_shuffle_epi8(bytes, shuffle), shift),
+ mask));
+ }
+}
+
+using Group8Fn = void (*)(const uint8_t*, uint32_t*);
+
+template <size_t... B>
+constexpr std::array<Group8Fn, 33> makeGroup8Table(std::index_sequence<B...>) {
+ return {{(B >= 1 && B <= kMaxShuffledWidth ? &unpackGroup8<static_cast<int>(B)> : nullptr)...}};
+}
+
+constexpr std::array<Group8Fn, 33> kUnpackGroup8 = makeGroup8Table(std::make_index_sequence<33> {});
+#else
+using Group8Fn = void (*)(const uint8_t*, uint32_t*);
+constexpr std::array<Group8Fn, 33> kUnpackGroup8 = {};
+#endif
+
+template <int B, int L>
+void unpackUnit(const uint8_t* in, uint32_t* out) {
+ if constexpr (B == 0) {
+ memset(out, 0, kGroup * L * sizeof(uint32_t));
+#ifdef __AVX2__
+ } else if constexpr (L == 1 && B <= kMaxShuffledWidth) {
+ for (int g = 0; g < 4; ++g) {
+ unpackGroup8<B>(in + g * B, out + 8 * g);
+ }
+#endif
+ } else {
+ uint32_t w[B * L];
+ memcpy(w, in, sizeof(w));
+ constexpr uint32_t mask = B == 32 ? ~0u : (1u << (B % 32)) - 1;
+ PFOR_UNROLL
+ for (int i = 0; i < 32; ++i) {
+ const int bit = i * B;
+ const int j = bit >> 5;
+ const int s = bit & 31;
+ for (int l = 0; l < L; ++l) {
+ uint32_t v = w[j * L + l] >> s;
+ if (s + B > 32) {
+ v |= w[(j + 1) * L + l] << (32 - s);
+ }
+ out[i * L + l] = v & mask;
+ }
+ }
+ }
+}
+
+using UnpackFn = void (*)(const uint8_t*, uint32_t*);
+
+template <int L, size_t... B>
+constexpr std::array<UnpackFn, 33> makeUnpackTable(std::index_sequence<B...>) {
+ return {{&unpackUnit<static_cast<int>(B), L>...}};
+}
+
+constexpr std::array<UnpackFn, 33> kUnpackHorizontal =
+ makeUnpackTable<1>(std::make_index_sequence<33> {});
+constexpr std::array<UnpackFn, 33> kUnpackVertical =
+ makeUnpackTable<kLanes>(std::make_index_sequence<33> {});
+
+// The inverse of unpackUnit: packs 32 rows x L lanes (values at i*L+l) into B*L words.
+template <int B, int L>
+void packUnit(const uint32_t* v, uint32_t* words) {
+ if constexpr (B != 0) {
+ uint32_t w[B * L] = {};
+ constexpr uint32_t mask = B == 32 ? ~0u : (1u << (B % 32)) - 1;
+ PFOR_UNROLL
+ for (int i = 0; i < 32; ++i) {
+ const int bit = i * B;
+ const int j = bit >> 5;
+ const int s = bit & 31;
+ for (int l = 0; l < L; ++l) {
+ const uint32_t x = v[i * L + l] & mask;
+ w[j * L + l] |= x << s;
+ if (s + B > 32) {
+ w[(j + 1) * L + l] |= x >> (32 - s);
+ }
+ }
+ }
+ memcpy(words, w, sizeof(w));
+ }
+}
+
+using PackFn = void (*)(const uint32_t*, uint32_t*);
+
+template <int L, size_t... B>
+constexpr std::array<PackFn, 33> makePackTable(std::index_sequence<B...>) {
+ return {{&packUnit<static_cast<int>(B), L>...}};
+}
+
+constexpr std::array<PackFn, 33> kPackHorizontal =
+ makePackTable<1>(std::make_index_sequence<33> {});
+constexpr std::array<PackFn, 33> kPackVertical =
+ makePackTable<kLanes>(std::make_index_sequence<33> {});
+
+inline size_t packedSize(size_t k, uint32_t b, bool vertical) {
+ return vertical ? kGroup * b : (k * b + 7) / 8;
+}
+
+// Writes k values of width b, returns the number of bytes written.
+size_t packValues(const uint32_t* t, size_t k, uint32_t b, bool vertical, uint8_t* out) {
+ if (b == 0) {
+ return 0;
+ }
+ if (vertical) {
+ uint32_t words[kLanes * 32];
+ kPackVertical[b](t, words);
+ memcpy(out, words, kGroup * b);
+ return kGroup * b;
+ }
+ uint8_t* o = out;
+ for (size_t g = 0; g < k; g += kGroup) {
+ const size_t n = std::min(kGroup, k - g);
+ uint32_t words[32];
+ if (n == kGroup) {
+ kPackHorizontal[b](t + g, words);
+ } else {
+ uint32_t buf[kGroup] = {};
+ memcpy(buf, t + g, n * sizeof(uint32_t));
+ kPackHorizontal[b](buf, words);
+ }
+ const size_t bytes = n == kGroup ? 4 * b : (n * b + 7) / 8;
+ memcpy(o, words, bytes);
+ o += bytes;
+ }
+ return o - out;
+}
+
+// ---------------------------------------------------------------------------------
+// Variable length integers
+// ---------------------------------------------------------------------------------
+
+constexpr uint32_t kVarint1 = 177;
+constexpr uint32_t kVarint2 = kVarint1 + (64u << 8);
+constexpr uint32_t kVarint3 = kVarint2 + (8u << 16);
+
+uint8_t* putFirst(uint8_t* o, uint32_t x) {
+ if (x < (1u << 7)) {
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < (1u << 14)) {
+ *o++ = static_cast<uint8_t>(0x80 | (x >> 8));
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < (1u << 21)) {
+ *o++ = static_cast<uint8_t>(0xc0 | (x >> 16));
+ memcpy(o, &x, 2);
+ o += 2;
+ } else if (x < (1u << 28)) {
+ *o++ = static_cast<uint8_t>(0xe0 | (x >> 24));
+ *o++ = static_cast<uint8_t>(x >> 16);
+ *o++ = static_cast<uint8_t>(x >> 8);
+ *o++ = static_cast<uint8_t>(x);
+ } else {
+ *o++ = 0xf0;
+ memcpy(o, &x, 4);
+ o += 4;
+ }
+ return o;
+}
+
+// The hi parts of a list block are either that many variable length integers or, if that
+// is smaller, the marker kRawList followed by plain 32-bit values.
+constexpr uint32_t kRawList = 0xff;
+constexpr uint32_t kVarint4 = 1u << 24;
+
+uint8_t* putVarint(uint8_t* o, uint32_t x) {
+ if (x < kVarint1) {
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < kVarint2) {
+ x -= kVarint1;
+ *o++ = static_cast<uint8_t>(0xb1 + (x >> 8));
+ *o++ = static_cast<uint8_t>(x);
+ } else if (x < kVarint3) {
+ x -= kVarint2;
+ *o++ = static_cast<uint8_t>(0xf1 + (x >> 16));
+ memcpy(o, &x, 2);
+ o += 2;
+ } else if (x < kVarint4) {
+ *o++ = 0xf9;
+ memcpy(o, &x, 3);
+ o += 3;
+ } else {
+ *o++ = 0xfa;
+ memcpy(o, &x, 4);
+ o += 4;
+ }
+ return o;
+}
+
+// Number of t[i] >= threshold. Written as a plain count so that it is vectorized.
+inline size_t countAtLeast(const uint32_t* t, size_t k, uint64_t threshold) {
+ if (threshold > UINT32_MAX) {
+ return 0;
+ }
+ const uint32_t limit = static_cast<uint32_t>(threshold);
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ n += t[i] >= limit;
+ }
+ return n;
+}
+
+// Bytes of the hi parts (t >> b) of the `exceptions` values wider than b, written as variable
+// length integers; maxBits is the width of the widest value.
+inline size_t hiVarintBytes(const uint32_t* t, size_t k, uint32_t b, uint32_t maxBits,
+ size_t exceptions) {
+ if (maxBits - b <= 7) {
+ return exceptions; // every hi part is below 128, one byte each
+ }
+ return exceptions + countAtLeast(t, k, static_cast<uint64_t>(kVarint1) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint2) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint3) << b) +
+ countAtLeast(t, k, static_cast<uint64_t>(kVarint4) << b);
+}
+
+// Size of the hi parts section of a list block with n exceptions; sets `raw` if the raw form
+// is the smaller.
+inline size_t hiListSize(size_t varints, size_t n, bool& raw) {
+ const size_t rawSize = 1 + 4 * n;
+ raw = rawSize < varints;
+ return raw ? rawSize : varints;
+}
+
+// ---------------------------------------------------------------------------------
+// Block encoder
+// ---------------------------------------------------------------------------------
+
+// Encodes the k deltas t[0..k) as one block and returns the number of bytes written.
+size_t encodeBlock(const uint32_t* t, size_t k, bool vertical, uint8_t* out) {
+ uint8_t* o = out;
+
+ if (std::all_of(t, t + k, [&](uint32_t x) { return x == t[0]; })) {
+ if (t[0] == 0) {
+ *o++ = kModePlain << 6;
+ } else {
+ const uint32_t b = bitLength(t[0]);
+ *o++ = static_cast<uint8_t>(kModeConst << 6 | b);
+ memcpy(o, &t[0], (b + 7) / 8);
+ o += (b + 7) / 8;
+ }
+ return o - out;
+ }
+
+ uint32_t any = 0;
+ for (size_t i = 0; i < k; ++i) {
+ any |= t[i];
+ }
+ const uint32_t maxBits = bitLength(any);
+
+ // Choose the packed width and the way the values wider than it are patched.
+ struct Plan {
+ size_t cost;
+ uint32_t mode;
+ uint32_t b;
+ };
+ Plan best = {1 + packedSize(k, maxBits, vertical), kModePlain, maxBits};
+ size_t fewest = 0; // the exceptions at the previous (wider) b; there are at least as many now
+ for (int64_t b = static_cast<int64_t>(maxBits) - 1; b >= 0; --b) {
+ const uint32_t bx = maxBits - static_cast<uint32_t>(b);
+ const size_t packed = packedSize(k, static_cast<uint32_t>(b), vertical);
+
+ // Neither the list (2 bytes + a byte per position and per hi part) nor the bitmap can be
+ // smaller than this, so this width cannot win and its exceptions need not be counted.
+ if (2 + packed + std::min(2 * fewest, (k + 7) / 8 + (fewest * bx + 7) / 8) >= best.cost) {
+ continue;
+ }
+ const size_t exceptions = countAtLeast(t, k, uint64_t(1) << b);
+ fewest = exceptions;
+
+ const size_t bitmapCost = 2 + (k + 7) / 8 + (exceptions * bx + 7) / 8 + packed;
+ if (bitmapCost < best.cost) {
+ best = {bitmapCost, kModeBitmap, static_cast<uint32_t>(b)};
+ }
+ const size_t listBase = 2 + packed + exceptions;
+ if (exceptions <= 255 && listBase + exceptions < best.cost) {
+ bool raw;
+ const size_t cost = listBase + hiListSize(hiVarintBytes(t, k, static_cast<uint32_t>(b),
+ maxBits, exceptions),
+ exceptions, raw);
+ if (cost < best.cost) {
+ best = {cost, kModeList, static_cast<uint32_t>(b)};
+ }
+ }
+ }
+
+ const uint32_t b = best.b;
+ *o++ = static_cast<uint8_t>(best.mode << 6 | b);
+ if (best.mode == kModePlain) {
+ o += packValues(t, k, b, vertical, o);
+ } else if (best.mode == kModeList) {
+ uint8_t* count = o++;
+ o += packValues(t, k, b, vertical, o);
+ uint8_t positions[256];
+ uint32_t hi[256];
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ if (t[i] >> b != 0) {
+ hi[n] = t[i] >> b;
+ positions[n++] = static_cast<uint8_t>(i);
+ }
+ }
+ bool raw;
+ hiListSize(hiVarintBytes(t, k, b, maxBits, n), n, raw);
+ if (raw) {
+ *o++ = kRawList;
+ memcpy(o, hi, 4 * n);
+ o += 4 * n;
+ } else {
+ for (size_t i = 0; i < n; ++i) {
+ o = putVarint(o, hi[i]);
+ }
+ }
+ *count = static_cast<uint8_t>(n);
+ memcpy(o, positions, n);
+ o += n;
+ } else {
+ *o++ = static_cast<uint8_t>(maxBits - b);
+ uint8_t* bitmap = o;
+ memset(bitmap, 0, (k + 7) / 8);
+ o += (k + 7) / 8;
+ uint32_t hi[256];
+ size_t n = 0;
+ for (size_t i = 0; i < k; ++i) {
+ if (t[i] >> b != 0) {
+ bitmap[i >> 3] |= static_cast<uint8_t>(1u << (i & 7));
+ hi[n++] = t[i] >> b;
+ }
+ }
+ o += packValues(hi, n, maxBits - b, false, o);
+ o += packValues(t, k, b, vertical, o);
+ }
+ return o - out;
+}
+
+// ---------------------------------------------------------------------------------
+// Block decoder
+// ---------------------------------------------------------------------------------
+
+struct Reader {
+ const uint8_t* p;
+ const uint8_t* end;
+
+ void need(size_t n) const {
+ if (static_cast<size_t>(end - p) < n) {
+ corrupt("PFOR: truncated block");
+ }
+ }
+ uint8_t u8() {
+ need(1);
+ return *p++;
+ }
+};
+
+uint32_t getFirst(Reader& r) {
+ const uint32_t b0 = r.u8();
+ if (b0 < 0x80) {
+ return b0;
+ }
+ if (b0 < 0xc0) {
+ return ((b0 & 0x3f) << 8) | r.u8();
+ }
+ if (b0 < 0xe0) {
+ r.need(2);
+ const uint32_t v = ((b0 & 0x1f) << 16) | load32(r.p, 2);
+ r.p += 2;
+ return v;
+ }
+ if (b0 < 0xf0) {
+ r.need(3);
+ const uint32_t v = ((b0 & 0x0f) << 24) | (static_cast<uint32_t>(r.p[0]) << 16) |
+ (static_cast<uint32_t>(r.p[1]) << 8) | r.p[2];
+ r.p += 3;
+ return v;
+ }
+ if (b0 != 0xf0) {
+ corrupt("PFOR: bad first value");
+ }
+ r.need(4);
+ const uint32_t v = load32(r.p, 4);
+ r.p += 4;
+ return v;
+}
+
+uint32_t getVarint(Reader& r) {
+ const uint32_t b0 = r.u8();
+ if (b0 < 0xb1) {
+ return b0;
+ }
+ if (b0 <= 0xf0) {
+ return kVarint1 + (((b0 - 0xb1) << 8) | r.u8());
+ }
+ if (b0 <= 0xf8) {
+ r.need(2);
+ const uint32_t v = kVarint2 + (((b0 - 0xf1) << 16) | load32(r.p, 2));
+ r.p += 2;
+ return v;
+ }
+ if (b0 == 0xf9) {
+ r.need(3);
+ const uint32_t v = load32(r.p, 3);
+ r.p += 3;
+ return v;
+ }
+ if (b0 == 0xfa) {
+ r.need(4);
+ const uint32_t v = load32(r.p, 4);
+ r.p += 4;
+ return v;
+ }
+ corrupt("PFOR: bad exception value");
+}
+
+// Reads k packed values of width b.
+void unpackValues(Reader& r, size_t k, uint32_t b, bool vertical, uint32_t* t) {
+ const size_t bytes = packedSize(k, b, vertical);
+ r.need(bytes);
+ if (vertical) {
+ kUnpackVertical[b](r.p, t);
+ } else {
+ // Units are read with kUnpackOverread bytes more than they have, which kDecodePadding
+ // allows past the end.
+ static_assert(kUnpackOverread <= kDecodePadding, "see PForCodec.h");
+ static_assert(8 <= kDecodePadding, "see PForCodec.h");
+ size_t g = 0;
+ const uint8_t* p = r.p;
+ for (; g + kGroup <= k; g += kGroup, p += 4 * b) {
+ kUnpackHorizontal[b](p, t + g);
+ }
+ // fewer than 32 values left: groups of 8 as far as possible, then each value comes from
+ // the 8 bytes at its first byte
+ size_t i = 0;
+ if (const Group8Fn group8 = kUnpackGroup8[b]) {
+ for (; g + i + 8 <= k; i += 8) {
+ group8(p + (i / 8) * b, t + g + i);
+ }
+ }
+ const uint32_t mask = b >= 32 ? ~0u : (1u << b) - 1;
+ for (; g + i < k; ++i) {
+ uint64_t word;
+ memcpy(&word, p + ((i * b) >> 3), sizeof(word));
+ t[g + i] = static_cast<uint32_t>(word >> ((i * b) & 7)) & mask;
+ }
+ }
+ r.p += bytes;
+}
+
+#ifdef __AVX2__
+// For a byte of the exception bitmap, the index into the (still packed) hi parts of the value of
+// each lane: the number of exceptions before that lane.
+struct ExpandTable {
+ alignas(32) uint32_t index[256][8];
+ alignas(32) uint32_t mask[256][8]; // all ones for the lanes whose bit is set
+};
+
+constexpr ExpandTable makeExpandTable() {
+ ExpandTable t = {};
+ for (int bits = 0; bits < 256; ++bits) {
+ uint32_t before = 0;
+ for (int lane = 0; lane < 8; ++lane) {
+ t.index[bits][lane] = before;
+ t.mask[bits][lane] = ((bits >> lane) & 1) ? ~0u : 0u;
+ before += (bits >> lane) & 1;
+ }
+ }
+ return t;
+}
+
+constexpr ExpandTable kExpandTable = makeExpandTable();
+#endif
+
+// t[i] |= hi[n] << b for the n-th set bit i of the bitmap. `hi` must have 8 readable entries
+// after the last exception.
+inline void applyBitmapExceptions(uint32_t* t, const uint8_t* bitmap, size_t k, const uint32_t* hi,
+ uint32_t b) {
+ size_t n = 0;
+ size_t byte = 0;
+#ifdef __AVX2__
+ // where the hi parts of each byte start; computed first so that the bytes below do not wait
+ // for each other
+ uint32_t before[33];
+ before[0] = 0;
+ for (size_t j = 0; j < k / 8; ++j) {
+ before[j + 1] = before[j] + __builtin_popcount(bitmap[j]);
+ }
+ const __m128i shift = _mm_cvtsi32_si128(static_cast<int>(b));
+ for (; byte < k / 8; ++byte) {
+ const uint32_t bits = bitmap[byte];
+ const __m256i spread = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(hi + before[byte])),
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(kExpandTable.index[bits])));
+ __m256i* dst = reinterpret_cast<__m256i*>(t + 8 * byte);
+ _mm256_storeu_si256(
+ dst, _mm256_or_si256(
+ _mm256_loadu_si256(dst),
+ _mm256_and_si256(_mm256_sll_epi32(spread, shift),
+ _mm256_loadu_si256(reinterpret_cast<const __m256i*>(
+ kExpandTable.mask[bits])))));
+ }
+ n = before[k / 8];
+#endif
+ for (; byte < (k + 7) / 8; ++byte) {
+ for (uint32_t bits = bitmap[byte]; bits != 0; bits &= bits - 1) {
+ const size_t i = byte * 8 + __builtin_ctz(bits);
+ if (i >= k) {
+ corrupt("PFOR: bad exception position");
+ }
+ t[i] |= hi[n++] << b;
+ }
+ }
+}
+
+// Decodes the next block of k deltas into t.
+void decodeBlock(Reader& r, size_t k, bool vertical, uint32_t* t) {
+ const uint32_t header = r.u8();
+ const uint32_t mode = header >> 6;
+ const uint32_t b = header & 0x3f;
+ if (b > 32) {
+ corrupt("PFOR: bad bit width");
+ }
+
+ if (mode == kModePlain) {
+ unpackValues(r, k, b, vertical, t);
+ } else if (mode == kModeConst) {
+ const size_t bytes = (b + 7) / 8;
+ r.need(bytes);
+ static_assert(4 <= kDecodePadding, "see PForCodec.h");
+ // always load 4 bytes, which is cheaper than a load of a variable size
+ const uint32_t v = load32(r.p, 4) & (bytes >= 4 ? ~0u : (1u << (8 * bytes)) - 1);
+ r.p += bytes;
+ std::fill(t, t + k, v);
+ } else if (mode == kModeList) {
+ if (b >= 32) {
+ corrupt("PFOR: bad bit width");
+ }
+ const size_t count = r.u8();
+ if (count > k) {
+ corrupt("PFOR: bad exception count");
+ }
+ unpackValues(r, k, b, vertical, t);
+ uint32_t hi[256];
+ if (count > 0) {
+ r.need(1);
+ if (*r.p == kRawList) {
+ ++r.p;
+ r.need(4 * count);
+ memcpy(hi, r.p, 4 * count);
+ r.p += 4 * count;
+ } else {
+ for (size_t i = 0; i < count; ++i) {
+ hi[i] = getVarint(r);
+ }
+ }
+ }
+ r.need(count);
+ for (size_t i = 0; i < count; ++i) {
+ const size_t pos = r.p[i];
+ if (pos >= k) {
+ corrupt("PFOR: bad exception position");
+ }
+ t[pos] |= hi[i] << b;
+ }
+ r.p += count;
+ } else {
+ const uint32_t bx = r.u8();
+ if (b >= 32 || bx == 0 || b + bx > 32) {
+ corrupt("PFOR: bad bit width");
+ }
+ const size_t bitmapBytes = (k + 7) / 8;
+ r.need(bitmapBytes);
+ const uint8_t* bitmap = r.p;
+ r.p += bitmapBytes;
+ size_t count = 0;
+ for (size_t i = 0; i < bitmapBytes; ++i) {
+ count += __builtin_popcount(bitmap[i]);
+ }
+ uint32_t hi[256 + 8];
+ unpackValues(r, count, bx, false, hi);
+ memset(hi + count, 0, 8 * sizeof(uint32_t)); // the vector code reads whole registers
+ unpackValues(r, k, b, vertical, t);
+ applyBitmapExceptions(t, bitmap, k, hi, b);
+ }
+}
+
+// ---------------------------------------------------------------------------------
+// Whole stream
+// ---------------------------------------------------------------------------------
+
+template <class Transform, size_t kBlock, bool kVertical>
+size_t encodeImpl(const uint32_t* in, size_t n, uint8_t* out) {
+ if (n == 0) {
+ return 0;
+ }
+ uint8_t* o = putFirst(out, in[0]);
+ uint32_t t[kBlock];
+ for (size_t i = 1; i < n;) {
+ const size_t k = std::min(kBlock, n - i);
+ for (size_t j = 0; j < k; ++j) {
+ t[j] = Transform::forward(in[i + j], in[i + j - 1]);
+ }
+ o += encodeBlock(t, k, kVertical && k == kBlock, o);
+ i += k;
+ }
+ return o - out;
+}
+
+// Inclusive prefix sum within the 8 lanes.
+inline U32x8 scan8(U32x8 x) {
+ const U32x8 zero = {};
+ x += PFOR_SHUFFLE(x, zero, 8, 0, 1, 2, 3, 4, 5, 6);
+ x += PFOR_SHUFFLE(x, zero, 8, 8, 0, 1, 2, 3, 4, 5);
+ x += PFOR_SHUFFLE(x, zero, 8, 8, 8, 8, 0, 1, 2, 3);
+ return x;
+}
+
+// Turns the k transformed values t into out[i] = cur + delta(t[0]) + ... + delta(t[i]) and
+// returns the last value. Eight values are scanned at a time; the scan of a group does not
+// depend on the running total, which only costs one add per group.
+template <class Transform>
+inline uint32_t accumulate(uint32_t cur, const uint32_t* t, size_t k, uint32_t* out) {
+ size_t i = 0;
+ if (k >= 8) {
+ U32x8 total = {cur, cur, cur, cur, cur, cur, cur, cur};
+ for (; i + 8 <= k; i += 8) {
+ U32x8 x = scan8(Transform::toDelta(*reinterpret_cast<const U32x8Unaligned*>(t + i)));
+ *reinterpret_cast<U32x8Unaligned*>(out + i) = x + total;
+ total += PFOR_SHUFFLE(x, x, 7, 7, 7, 7, 7, 7, 7, 7);
+ }
+ cur = total[0];
+ }
+ for (; i < k; ++i) {
+ cur += Transform::toDelta(t[i]);
+ out[i] = cur;
+ }
+ return cur;
+}
+
+// Everything it calls is inlined: the arguments are few and the reader state stays in registers.
+template <class Transform, size_t kBlock, bool kVertical>
+__attribute__((flatten)) size_t decodeImpl(const uint8_t* in, const uint8_t* end, size_t n,
+ uint32_t* out) {
+ if (n == 0) {
+ return 0;
+ }
+ Reader r = {in, end};
+ uint32_t cur = getFirst(r);
+ out[0] = cur;
+ alignas(32) uint32_t t[kBlock];
+ for (size_t i = 1; i < n;) {
+ const size_t k = std::min(kBlock, n - i);
+ decodeBlock(r, k, kVertical && k == kBlock, t);
+ cur = accumulate<Transform>(cur, t, k, out + i);
+ i += k;
+ }
+ return r.p - in;
+}
+
+} // namespace
+
+size_t encodeZigzag128(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<ZigzagDelta, 128, false>(in, n, out);
+}
+size_t encodeZigzag256(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<ZigzagDelta, 256, true>(in, n, out);
+}
+size_t encodeDelta1_128(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<Delta1, 128, false>(in, n, out);
+}
+size_t encodeDelta1_256(const uint32_t* in, size_t n, uint8_t* out) {
+ return encodeImpl<Delta1, 256, true>(in, n, out);
+}
+
+size_t decodeZigzag128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<ZigzagDelta, 128, false>(in, end, n, out);
+}
+size_t decodeZigzag256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<ZigzagDelta, 256, true>(in, end, n, out);
+}
+size_t decodeDelta1_128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<Delta1, 128, false>(in, end, n, out);
+}
+size_t decodeDelta1_256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out) {
+ return decodeImpl<Delta1, 256, true>(in, end, n, out);
+}
+
+} // namespace pfor
+} // namespace util
+} // namespace lucene
diff --git a/src/core/CLucene/util/PForCodec.h b/src/core/CLucene/util/PForCodec.h
new file mode 100644
index 00000000000..e705b1f5268
--- /dev/null
+++ b/src/core/CLucene/util/PForCodec.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Patched frame-of-reference codec for uint32 arrays, used by the inverted index
+// postings. This is an independent implementation of the on-disk block format that
+// has been written into existing indexes (CodeMode kPfor / kPfor128 / kPfor256), so
+// that those indexes stay readable and newly written ones stay readable by old
+// versions.
+//
+// Stream layout (all multi-byte fields little endian unless noted):
+//
+// stream := first-value block*
+// first-value : the array's first element, 1..5 bytes
+// x < 2^7 : x
+// x < 2^14 : 0x80|x>>8, x
+// x < 2^21 : 0xc0|x>>16, x[16]
+// x < 2^28 : 0xe0|x>>24, x>>16, x>>8, x (big endian tail)
+// otherwise : 0xf0, x[32]
+//
+// The remaining n-1 elements are turned into n-1 "deltas" t[i] (see Transform below)
+// and cut into blocks of kBlock deltas; the last block holds the remainder. Every
+// block starts with a header byte h = mode<<6 | b, where b (0..32) is the bit width
+// of the packed part:
+//
+// mode 0 (plain) : packed(b)
+// mode 1 (list) : count, packed(b) of the low b bits, his, count x pos
+// his holds the hi parts (x >> b) of the exceptions: count variable
+// length integers, or the byte 0xff followed by count plain 32-bit
+// values. pos is the index in the block (1 byte).
+// mode 2 (bitmap) : bx, bitmap[ceil(k/8)], packed(bx) of the hi parts (x >> b) of the
+// flagged values in index order, packed(b) of the low b bits
+// mode 3 (const) : value[ceil(b/8)], all k deltas equal to value
+//
+// packed(w) of k values is LSB-first bit packing. It is "horizontal" (value i at bit
+// i*w) except for a full 256 value block of the 256 layout, which is "vertical":
+// value i lives in lane i%8, row i/8; each lane packs its 32 rows into w 32-bit words
+// and word j of lane l is stored at word index j*8+l.
+//
+// Variable length integer for hi parts:
+// x < 177 : x
+// x < 177+2^14 : 0xb1+(x-177)>>8, (x-177)
+// x < 16561+2^19 : 0xf1+(x-16561)>>16, (x-16561)[16]
+// x < 2^24 : 0xf9, x[24]
+// otherwise : 0xfa, x[32]
+namespace lucene {
+namespace util {
+namespace pfor {
+
+// Upper bound of the encoded size of n values, for sizing output buffers.
+constexpr size_t encodedSizeBound(size_t n) {
+ return 8 + n * 4 + n / 128 + 1;
+}
+
+// Delta = zigzag(in[i] - in[i-1]).
+size_t encodeZigzag128(const uint32_t* in, size_t n, uint8_t* out);
+size_t encodeZigzag256(const uint32_t* in, size_t n, uint8_t* out);
+// Delta = in[i] - in[i-1] - 1 (strictly increasing sequences).
+size_t encodeDelta1_128(const uint32_t* in, size_t n, uint8_t* out);
+size_t encodeDelta1_256(const uint32_t* in, size_t n, uint8_t* out);
+
+// The decoders may load up to kDecodePadding bytes past `end` (the loaded values are never
+// used), so the buffer must be readable that far.
+constexpr size_t kDecodePadding = 16;
+
+// Decoders write exactly n values to out and return the number of input bytes consumed.
+// Nothing at or after `end` is interpreted; a malformed or truncated stream throws
+// CL_ERR_CorruptIndex.
+size_t decodeZigzag128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeZigzag256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeDelta1_128(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+size_t decodeDelta1_256(const uint8_t* in, const uint8_t* end, size_t n, uint32_t* out);
+
+} // namespace pfor
+} // namespace util
+} // namespace lucene
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index b9a09bb3065..fba964a9595 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -56,6 +56,7 @@ SET(clucene_core_Files
./CLucene/util/NumericUtils.cpp
./CLucene/util/stringUtil.cpp
./CLucene/util/PFORUtil.cpp
+ ./CLucene/util/PForCodec.cpp
./CLucene/queryParser/FastCharStream.cpp
./CLucene/queryParser/MultiFieldQueryParser.cpp
./CLucene/queryParser/QueryParser.cpp
@@ -213,7 +214,6 @@ file(GLOB_RECURSE HEADERS ${clucene-core_SOURCE_DIR}/*.h )
#create the libraries
INCLUDE_DIRECTORIES(${clucene_SOURCE_DIR}/src/core)
-INCLUDE_DIRECTORIES(${clucene_SOURCE_DIR}/src/ext/for)
IF (BUILD_SHARED_LIBRARIES)
add_library(clucene-core SHARED
diff --git a/src/demo/CMakeLists.txt b/src/demo/CMakeLists.txt
index 171e554a9f3..5b054832f16 100644
--- a/src/demo/CMakeLists.txt
+++ b/src/demo/CMakeLists.txt
@@ -37,12 +37,10 @@ ADD_EXECUTABLE(cl_demo_index EXCLUDE_FROM_ALL
#./TestAnalyzer.cpp
#${demo_HEADERS}
#)
-#add_library(ic STATIC IMPORTED)
-#set_target_properties(ic PROPERTIES IMPORTED_LOCATION "${clucene-ext_SOURCE_DIR}/for/libic.a")
-TARGET_LINK_LIBRARIES(cl_demo clucene-core-static clucene-shared-static ic ${EXTRA_LIBS})
-TARGET_LINK_LIBRARIES(cl_demo_index clucene-core-static clucene-shared-static ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(cl_demo clucene-core-static clucene-shared-static ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(cl_demo_index clucene-core-static clucene-shared-static ${EXTRA_LIBS})
ADD_EXECUTABLE(allIndex EXCLUDE_FROM_ALL
./allIndex.cpp
@@ -51,7 +49,7 @@ ADD_EXECUTABLE(allIndex EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(allIndex clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(allIndex clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(abnormal EXCLUDE_FROM_ALL
./abnormal.cpp
@@ -60,7 +58,7 @@ ADD_EXECUTABLE(abnormal EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(abnormal clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(abnormal clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(search EXCLUDE_FROM_ALL
./search.cpp
@@ -68,7 +66,7 @@ ADD_EXECUTABLE(search EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(search clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(search clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(strSearch EXCLUDE_FROM_ALL
./strSearch.cpp
@@ -77,7 +75,7 @@ ADD_EXECUTABLE(strSearch EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(strSearch clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(strSearch clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(pointSearch4 EXCLUDE_FROM_ALL
./pointSearch4.cpp
@@ -86,7 +84,7 @@ ADD_EXECUTABLE(pointSearch4 EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(pointSearch4 clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(pointSearch4 clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(pointSearch8 EXCLUDE_FROM_ALL
./pointSearch8.cpp
@@ -95,7 +93,7 @@ ADD_EXECUTABLE(pointSearch8 EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(pointSearch8 clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
+TARGET_LINK_LIBRARIES(pointSearch8 clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
ADD_EXECUTABLE(demo EXCLUDE_FROM_ALL
./demo.cpp
@@ -104,4 +102,4 @@ ADD_EXECUTABLE(demo EXCLUDE_FROM_ALL
${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp
${demo_HEADERS}
)
-TARGET_LINK_LIBRARIES(demo clucene-core-static clucene-shared-static clucene-contribs-lib ic ${EXTRA_LIBS})
\ No newline at end of file
+TARGET_LINK_LIBRARIES(demo clucene-core-static clucene-shared-static clucene-contribs-lib ${EXTRA_LIBS})
\ No newline at end of file
diff --git a/src/ext/for/CMakeLists.txt b/src/ext/for/CMakeLists.txt
deleted file mode 100644
index 3b14781f69e..00000000000
--- a/src/ext/for/CMakeLists.txt
+++ /dev/null
@@ -1,91 +0,0 @@
-cmake_minimum_required(VERSION 3.10)
-project(powturbo)
-
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_C_STANDARD 99)
-
-if(NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Release)
-endif()
-
-set(DEBUG "-DNDEBUG -g -O3")
-set(OPT "-w -Wall -fstrict-aliasing -falign-loops -Wno-int-conversion")
-
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
- set(SSE "-march=corei7-avx -mtune=corei7-avx")
- set(AVX2 "-march=haswell")
- set(CMAKE_C_FLAGS ${SSE})
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
- set(SSE "-march=armv8-a")
- set(CMAKE_C_FLAGS "-march=armv8-a")
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
- set(SSE "-D__SSSE3__")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power9 -mtune=power9")
-endif()
-
-if(FLOAT16)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_FLOAT16")
-endif()
-
-if(STATIC)
- set(CMAKE_EXE_LINKER_FLAGS "-static")
-endif()
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${DEBUG} ${OPT}")
-
-separate_arguments(avx2_c_flags_list UNIX_COMMAND "${DEBUG} ${OPT}")
-separate_arguments(c_flags_list UNIX_COMMAND "${CMAKE_C_FLAGS}")
-
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-
-set(SRC_FILES
- ${CMAKE_CURRENT_SOURCE_DIR}/bitpack.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bitunpack.c
- ${CMAKE_CURRENT_SOURCE_DIR}/vp4c.c
- ${CMAKE_CURRENT_SOURCE_DIR}/vp4d.c
- ${CMAKE_CURRENT_SOURCE_DIR}/transpose.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bitutil.c
-)
-
-add_library(ic STATIC
- v8.c
- vint.c
- trlec.c
- trled.c
- vsimple.c
- bitutil.c
- bitpack.c
- bitunpack.c
- vp4c.c
- vp4d.c
- transpose.c
-)
-
-foreach(SRC_FILE ${SRC_FILES})
- get_filename_component(SRC_NAME ${SRC_FILE} NAME_WE)
- set(SSE_OUTPUT ${SRC_NAME}_sse.o)
- set(AVX2_OUTPUT ${SRC_NAME}_avx2.o)
-
- add_custom_command(
- OUTPUT ${SSE_OUTPUT}
- COMMAND ${CMAKE_C_COMPILER} -DSSE2_ON ${c_flags_list} -c -o ${SSE_OUTPUT} ${SRC_FILE}
- DEPENDS ${SRC_FILE}
- )
-
- target_sources(ic PRIVATE ${SSE_OUTPUT})
-
- if(USE_AVX2)
- add_custom_command(
- OUTPUT ${AVX2_OUTPUT}
- COMMAND ${CMAKE_C_COMPILER} ${AVX2} -DAVX2_ON ${avx2_c_flags_list} -c -o ${AVX2_OUTPUT} ${SRC_FILE}
- DEPENDS ${SRC_FILE}
- )
-
- target_sources(ic PRIVATE ${AVX2_OUTPUT})
- endif()
-endforeach()
-
-set(LIB_DESTINATION ../)
-
-install(TARGETS ic
- DESTINATION ${LIB_DESTINATION}
- COMPONENT ext)
diff --git a/src/ext/for/README.md b/src/ext/for/README.md
deleted file mode 100644
index 31e8f25ebb4..00000000000
--- a/src/ext/for/README.md
+++ /dev/null
@@ -1,585 +0,0 @@
-TurboPFor: Fastest Integer Compression
-
-[//]: # ([![Build Status][travisBadge]][travisLink])
-[//]: # ([travisBadge]: https://api.travis-ci.com/powturbo/TurboPFor-Integer-Compression.svg?branch=master)
-[//]: # ([travisLink]: https://app.travis-ci.com/powturbo/TurboPFor-Integer-Compression)
-======================================
-* **TurboPFor: The synonym for "integer compression"**
- * **ALL** functions available for **AMD/Intel**, **64 bits ARMv8 NEON** Linux+MacOS/M1 & **Power9 Altivec**
- * 100% C (C++ headers), as simple as memcpy. OS:Linux amd64, arm64, Power9, MacOs (Amd/intel + Apple M1),
- * :+1: **Java** Critical Natives/JNI. Access TurboPFor **incl. SIMD/AVX2!** from Java as fast as calling from C
- * :sparkles: **FULL** range 8/16/32/64 bits scalar + 16/32/64 bits SIMD functions
- * No other "Integer Compression" compress/decompress faster
- * :sparkles: Direct Access, **integrated** (SIMD/AVX2) FOR/delta/Delta of Delta/Zigzag for sorted/unsorted arrays
- * **16 bits** + **64 bits** SIMD integrated functions
-* **For/PFor/PForDelta**
- * **Novel TurboPFor** (PFor/PForDelta) scheme w./ **direct access** + **SIMD/AVX2**. **+RLE**
- * Outstanding compression/speed. More efficient than **ANY** other fast "integer compression" scheme.
- * Compress 70 times faster and decompress up to 4 times faster than OptPFD
-* **Bit Packing**
- * Fastest and most efficient **"SIMD Bit Packing"** **15 Billions integers/sec (60Gb/s!)**
- * Scalar **"Bit Packing"** decoding nearly as fast as SIMD-Packing in realistic (No "pure cache") scenarios
- * **Direct/Random Access** : Access any single bit packed entry with **zero decompression**
-* **Variable byte**
- * Scalar **"Variable Byte"** faster and more efficient than **ANY** other implementation
- * SIMD **TurboByte** fastest group varint (16+32 bits) incl. integrated delta,zigzag,...
- * **TurboByte+TurboPackV** novel hybrid scheme combining the fastest SIMD codecs.
-* **Simple family**
- * **Novel** **"Variable Simple"** (incl. **RLE**) faster and more efficient than simple16, simple-8b
-* **Elias fano**
- * Fastest **"Elias Fano"** implementation w/ or w/o SIMD/AVX2
-+ **Transform**
- * Scalar & SIMD Transform: Delta, Zigzag, Zigzag of delta, XOR, Transpose/Shuffle,
- * **lossy** floating point compression with *TurboPFor* or [TurboTranspose](https://github.com/powturbo/TurboTranspose)+lz77
-* **Floating Point Compression**
- * Delta/Zigzag + improved gorilla style + (Differential) Finite Context Method FCM/DFCM floating point compression
- * Using **TurboPFor**, unsurpassed compression and more than 5 GB/s throughput
- * Point wise relative error bound **lossy** floating point compression
- * **TurboFloat** novel efficient floating point compression using TurboPFor
-* **Time Series Compression**
- * **Fastest Gorilla** 16/32/64 bits style compression (**zigzag of delta** + **RLE**).
- * can compress times series to only 0.01%. Speed > 10 GB/s compression and > 13 GB/s decompress.
-* **Inverted Index ...do less, go fast!**
- * Direct Access to compressed *frequency* and *position* data w/ zero decompression
- * **Novel** **"Intersection w/ skip intervals"**, decompress the minimum necessary blocks (**~10-15%)!**.
- * **Novel** Implicit skips with zero extra overhead
- * **Novel** Efficient **Bidirectional** Inverted Index Architecture (forward/backwards traversal) incl. "integer compression".
- * more than **2000! queries per second** on GOV2 dataset (25 millions documents) on a **SINGLE** core
- * :sparkles: Revolutionary Parallel Query Processing on Multicores **> 7000!!! queries/sec** on a simple quad core PC.<br>
- **...forget** ~~Map Reduce, Hadoop, multi-node clusters,~~ ...
-
-
-
-### Integer Compression Benchmark (single thread):
-- Download [IcApp](https://sites.google.com/site/powturbo/downloads) a new benchmark for TurboPFor<br>
- for testing allmost all integer and floating point file types.
-- Practical (No **PURE** cache) "integer compression" benchmark w/ **large** arrays.
-- [Benchmark Intel CPU: Skylake i7-6700 3.4GHz gcc 9.2](https://github.com/powturbo/TurboPFor/issues/47)
-- [Benchmark ARM: ARMv8 A73-ODROID-N2 1.8GHz](https://github.com/powturbo/TurboPFor/issues/49)
-
-##### - Synthetic data:
- - Generate and test (zipfian) skewed distribution (100.000.000 integers, Block size=128/256)<br>
- Note: Unlike general purpose compression, a small fixed size (ex. 128 integers) is in general used in "integer compression".
- Large blocks involved, while processing queries (inverted index, search engines, databases, graphs, in memory computing,...) need to be entirely decoded.
-
- ./icbench -a1.5 -m0 -M255 -n100M ZIPF
-
-|C Size|ratio%|Bits/Integer|C MB/s|D MB/s|Name 2019.11|
-|--------:|-----:|--------:|----------:|----------:|--------------|
-|62,939,886| 15.7| 5.04|**2369**|**10950**|**TurboPFor256**|
-|63,392,759| 15.8| 5.07|1359|7803|**TurboPFor128**|
-|63,392,801| 15.8| 5.07|1328|924|**TurboPForDA**|
-|65,060,504| 16.3| 5.20|60|2748|[FP_SIMDOptPFor](#FastPFor)|
-|65,359,916|16.3| 5.23| 32|2436|PC_OptPFD|
-|73,477,088|18.4| 5.88|408|2484|PC_Simple16|
-|73,481,096| 18.4| 5.88|624|8748|[FP_SimdFastPFor](#FastPFor) 64Ki *|
-|76,345,136| 19.1| 6.11|1072|2878|**VSimple**|
-|91,947,533| 23.0| 7.36|284|11737|[QMX](#QMX) 64k *|
-|93,285,864| 23.3| 7.46|1568|10232|[FP_GroupSimple](#FastPFor) 64Ki *|
-|95,915,096|24.0| 7.67| 848|3832|Simple-8b|
-|99,910,930| 25.0| 7.99|**17298**|**12408**|**TurboByte+TurboPack**|
-|99,910,930| 25.0| 7.99|**17357**|**12363**|**TurboPackV** sse|
-|99,910,930| 25.0| 7.99|11694|10138|**TurboPack** scalar|
-|99,910,930| 25.0| 7.99|8420|8876|**TurboFor**|
-|100,332,929| 25.1| 8.03|17077|11170|**TurboPack256V** avx2|
-|101,015,650| 25.3| 8.08|11191|10333|**TurboVByte**|
-|102,074,663| 25.5| 8.17|6689|9524|[MaskedVByte](#MaskedVByte)|
-|102,074,663| 25.5| 8.17|2260|4208|[PC_Vbyte](#PolyCom)|
-|102,083,036| 25.5| 8.17|5200|4268|[FP_VByte](#FastPFor)|
-|112,500,000| 28.1| 9.00|1528|12140|[VarintG8IU](#VarintG8IU)|
-|125,000,000| 31.2|10.00|13039|12366|**TurboByte**|
-|125,000,000| 31.2|10.00|11197|11984|[StreamVbyte 2019](#StreamVByte)|
-|400,000,000| 100.00| 32.00| 8960|8948|Copy|
-| | | | N/A | N/A |EliasFano|
-
-(*) codecs inefficient for small block sizes are tested with 64Ki integers/block.
-
-- MB/s: 1.000.000 bytes/second. **1000 MB/s = 1 GB/s**<br>
-- **#BOLD** = pareto frontier.<br>
-- FP=FastPFor SC:simdcomp PC:Polycom<br>
-- TurboPForDA,TurboForDA: Direct Access is normally used when accessing few individual values.<br>
-- Eliasfano can be directly used only for increasing sequences
-------------------------------------------------------------------------
-##### - Data files:
- - gov2.sorted from [DocId data set](#DocId) Block size=128/Delta coding
-
- ./icbench -fS -r gov2.sorted
-
-
-
-|Size |Ratio %|Bits/Integer|C Time MB/s|D Time MB/s|Function 2019.11|
-|-----------:|------:|-----:|-------:|-------:|---------------------|
-| 3,321,663,893| 13.9| 4.44|**1320**|**6088**|**TurboPFor**|
-| 3,339,730,557| 14.0| 4.47| 32| 2144|PC.OptPFD|
-| 3,350,717,959| 14.0| 4.48|**1536**|**7128**|**TurboPFor256**|
-| 3,501,671,314| 14.6| 4.68| 56| 2840|**VSimple**|
-| 3,768,146,467| 15.8| 5.04|**3228**| 3652|**EliasFanoV**|
-| 3,822,161,885| 16.0| 5.11| 572| 2444|PC_Simple16|
-| 4,411,714,936| 18.4| 5.90|**9304**|**10444**|**TurboByte+TurboPack**|
-| 4,521,326,518| 18.9| 6.05| 836| 3296|Simple-8b|
-| 4,649,671,427| 19.4| 6.22|3084| 3848|**TurboVbyte**|
-| 4,955,740,045| 20.7| 6.63|7064|10268|**TurboPackV**|
-| 4,955,740,045| 20.7| 6.63|5724| 8020|**TurboPack**|
-| 5,205,324,760| 21.8| 6.96|6952| 9488|SC_SIMDPack128|
-| 5,393,769,503| 22.5| 7.21|**14466**|**11902**|**TurboPackV256**|
-| 6,221,886,390| 26.0| 8.32|6668| 6952|**TurboFor**|
-| 6,221,886,390| 26.0| 8.32|6644| 2260|**TurboForDA**|
-| 6,699,519,000| 28.0| 8.96|1888| 1980|FP_Vbyte|
-| 6,700,989,563| 28.0| 8.96|2740| 3384|MaskedVByte|
-| 7,622,896,878| 31.9|10.20| 836| 4792|VarintG8IU|
-| 8,060,125,035| 33.7|11.50|8456| 9476|Streamvbyte 2019|
-| 8,594,342,216| 35.9|11.50|5228| 6376|libfor|
-|23,918,861,764|100.0|32.00|5824| 5924|Copy|
-
-Block size: 64Ki = 256k bytes. Ki=1024 Integers
-
-|Size |Ratio %|Bits/Integer|C Time MB/s|D Time MB/s|Function |
-|----------:|-----:|----:|------:|------:|---------------------|
-| 3,164,940,562| 13.2|**4.23**|**1344**|**6004**|**TurboPFor 64Ki**|
-| 3,273,213,464| 13.7| 4.38|**1496**|**7008**|**TurboPFor256 64Ki**|
-| 3,965,982,954| 16.6| 5.30|**1520**| 2452|[lz4](#lz4)+DT 64Ki|
-| 4,234,154,427| 17.7| 5.66| 436| 5672|qmx 64Ki|
-| 6,074,995,117| 25.4| 8.13| 1976| 2916|[blosc_lz4](#blosc) 64Ki|
-| 8,773,150,644| 36.7|11.74| 2548|5204|blosc_lz 64Ki|
-
-"lz4+DT 64Ki" = Delta+Transpose from TurboPFor + lz4<br>
-"blosc_lz4" internal lz4 compressor+vectorized shuffle
-
-##### - Time Series:
-- Test file [Timestamps: ts.txt(sorted)](https://github.com/zhenjl/encoding/tree/master/benchmark/data)
-
- ./icapp -Ft ts.txt -I15 -J15
-
-|Function |C MB/s| size |ratio%| D MB/s|Text
-|----------------|-----:|--------:|------:|------:|--------------------|
-|bvzenc32 |**10632**|45,909|0.008|**12823**|ZigZag|
-|bvzzenc32 |**8914**|56,713|0.010|**13499**|ZigZag Delta of delta|
-|vsenc32 |**12294**|140,400| 0.024 |12877 |Variable Simple|
-|p4nzenc256v32 | 1932| 596,018| 0.10 |13326 |TurboPFor256 ZigZag|
-|p4ndenc256v32 | 1961| 596,018| 0.10 |13339 |TurboPFor256 Delta|
-|bitndpack256v32 |**12564**|909,189| 0.16 |13505 |TurboPackV256 Delta|
-|p4nzenc32 | 1810| 1,159,633| 0.20 | 8502 |TurboPFor ZigZag|
-|p4nzenc128v32 | 1795| 1,159,633| 0.20 |13338 |TurboPFor ZigZag|
-|bitnzpack256v32 | 9651| 1,254,757| 0.22 |**13503**|TurboPackV256 ZigZag|
-|bitnzpack128v32 |10155| 1,472,804| 0.26 |13380 |TurboPackV ZigZag|
-|vbddenc32 | 6198| 18,057,296| 3.13 |10982 |TurboVByte Delta of delta|
-|memcpy |13397|577,141,992|100.00||
-
-##### - Transpose/Shuffle (no compression)
- ./icbench -eTRANSFORM ZIPF
-
-|Size |C Time MB/s|D Time MB/s|Function|
-|----------:|------:|------:|-----------------------------------|
-|100,000,000|**9400**|**9132**|**TPbyte 4** TurboPFor Byte Transpose/shuffle AVX2|
-|100,000,000|8784|8860|**TPbyte 4** TurboPFor Byte Transpose/shuffle SSE|
-|100,000,000|7688|7656|Blosc_Shuffle AVX2|
-|100,000,000|**5204**|**7460**|**TPnibble 4** TurboPFor Nibble Transpose/shuffle SSE|
-|100,000,000|6620|6284|Blosc shuffle SSE|
-|100,000,000|3156|3372|Bitshuffle AVX2|
-|100,000,000|2100|2176|Bitshuffle SSE|
-
-##### - (Lossy) Floating point compression:
- ./icapp -Fd file " 64 bits floating point raw file
- ./icapp -Ff file " 32 bits floating point raw file
- ./icapp -Fcf file " text file with miltiple entries (ex. 8.657,56.8,4.5 ...)
- ./icapp -Ftf file " text file (1 entry per line)
- ./icapp -Ftf file -v5 " + display the first entries read
- ./icapp -Ftf file.csv -K3 " but 3th column in a csv file (ex. number,Text,456.5 -> 456.5
- ./icapp -Ftf file -g.001 " lossy compression with allowed pointwise relative error 0.001
-
-- see also [TurboTranspose](https://github.com/powturbo/TurboTranspose)
-
-##### - Compressed Inverted Index Intersections with GOV2<br />
- GOV2: 426GB, 25 Millions documents, average doc. size=18k.
-
- + Aol query log: 18.000 queries<br />
- **~1300** queries per second (single core)<br />
- **~5000** queries per second (quad core)<br />
- Ratio = 14.37% Decoded/Total Integers.
-
- + TREC Million Query Track (1MQT):<br />
- **~1100** queries per second (Single core)<br />
- **~4500** queries per second (Quad core CPU)<br />
- Ratio = 11.59% Decoded/Total Integers.
-
-- Benchmarking intersections (Single core, AOL query log)
-
-| max.docid/q|Time s| q/s | ms/q | % docid found|
-|-----------------:|---:|----:|-----:|-------:|
-|1.000|7.88|2283.1|0.438|81|
-|10.000|10.54|1708.5|0.585|84|
-| ALL |13.96|1289.0|0.776|100|
-q/s: queries/second, ms/q:milliseconds/query
-
-- Benchmarking Parallel Query Processing (Quad core, AOL query log)
-
-| max.docid/q|Time s| q/s | ms/q | % docids found|
-|-----------------:|----:|----:|-----:|-------:|
-|1.000|2.66|6772.6|0.148|81|
-|10.000|3.39|5307.5|0.188|84|
-|ALL|3.57|5036.5|0.199|100|
-
-###### Notes:
-- Search engines are spending 90% of the time in intersections when processing queries.
-- Most search engines are using pruning strategies, caching popular queries,... to reduce the time for intersections and query processing.
-- As indication, google is processing [40.000 Queries per seconds](http://www.internetlivestats.com/google-search-statistics/),
-using [900.000 multicore servers](https://www.cloudyn.com/blog/10-facts-didnt-know-server-farms/) for searching [8 billions web pages](http://searchenginewatch.com/sew/study/2063479/coincidentally-googles-index-size-jumps) (320 X size of GOV2).
-- Recent "integer compression" GOV2 experiments (best paper at ECIR 2014) [On Inverted Index Compression for Search Engine Efficiency](http://www.dcs.gla.ac.uk/~craigm/publications/catena14compression.pdf) using 8-core Xeon PC are reporting 1.2 seconds per query (for 1.000 Top-k docids).
-
-### Compile:
- Download or clone TurboPFor
- git clone git://github.com/powturbo/TurboPFor.git
- cd TurboPFor
- make
-
-
- To benchmark external libraries + lz77 compression:
- git clone --recursive git://github.com/powturbo/TurboPFor.git
- cd TurboPFor
- make CODEC1=1 CODEC2=1 LZ=1
-
-###### Windows visual c++
- nmake /f makefile.vs
-
-###### Windows visual studio c++
- project files under vs/vs2017
-
-### Testing:
-##### - Synthetic data (use ZIPF parameter):
- + benchmark groups of "integer compression" functions <br />
-
- ./icbench -eBENCH -a1.2 -m0 -M255 -n100M ZIPF
- ./icbench -eBITPACK/VBYTE -a1.2 -m0 -M255 -n100M ZIPF
-
- >*Type "icbench -l1" for a list*
-
- >*-zipfian distribution alpha = 1.2 (Ex. -a1.0=uniform -a1.5=skewed distribution)<br />
- -number of integers = 100.000.000<br />
- -integer range from 0 to 255<br />*
-
- + Unsorted lists: individual function test (ex. Copy TurboPack TurboPFor)<br />
-
- ./icbench -a1.5 -m0 -M255 -ecopy/turbopack/turbopfor/turbopack256v ZIPF
-
- + Unsorted lists: Zigzag encoding w/ option **-fz** or FOR encoding<br />
-
- ./icbench -fz -eturbovbyte/turbopfor/turbopackv ZIPF
- ./icbench -eturboforv ZIPF
-
- + Sorted lists: differential coding w/ option **-fs** (increasing) or **-fS** (strictly increasing)<br />
-
- ./icbench -fs -eturbopack/turbopfor/turbopfor256v ZIPF
-
- + Generate interactive "file.html" plot for browsing
-
- ./icbench -p2 -S2 -Q3 file.tbb
-
- + Unit test: test function from bit size 0 to 32
-
- ./icbench -m0 -M32 -eturbpfor -fu
- ./icbench -m0 -M8 -eturbopack -fs -n1M
-
-##### - Data files:
- - Raw 32 bits binary data file [Test data](https://github.com/ot/partitioned_elias_fano/tree/master/test/test_data)
-
- ./icbench file
- ./icapp file
- ./icapp -Fs file "16 bits raw binary file
- ./icapp -Fu file "32 bits raw binary file
- ./icapp -Fl file "64 bits raw binary file
- ./icapp -Ff file "32 bits raw floating point binary file
- ./icapp -Fd file "64 bits raw floating point binary file
-
- - Text file: 1 entry per line. [Test data: ts.txt(sorted) and lat.txt(unsorted)](https://github.com/zhenjl/encoding/tree/master/benchmark/data))
-
- ./icbench -eBENCH -fts ts.txt
- ./icbench -eBENCH -ft lat.txt
-
- ./icapp -Fts data.txt "text file, one 16 bits integer per line
- ./icapp -Ftu ts.txt "text file, one 32 bits integer per line
- ./icapp -Ftl ts.txt "text file, one 64 bits integer per line
- ./icapp -Ftf file "text file, one 32 bits floating point (ex. 8.32456) per line
- ./icapp -Ftd file "text file, one 64 bits floating point (ex. 8.324567789) per line
- ./icapp -Ftd file -v5 "like prev., display the first 100 values read
- ./icapp -Ftd file -v5 -g.00001 "like prev., error bound lossy floating point compression
- ./icapp -Ftt file "text file, timestamp in seconds iso-8601 -> 32 bits integer (ex. 2018-03-12T04:31:06)
- ./icapp -FtT file "text file, timestamp in milliseconds iso-8601 -> 64 bits integer (ex. 2018-03-12T04:31:06.345)
- ./icapp -Ftl -D2 -H file "skip 1th line, convert numbers with 2 decimal digits to 64 bits integers (ex. 456.23 -> 45623)
- ./icapp -Ftl -D2 -H -K3 file.csv "like prev., use the 3th number in the line (ex. label=3245, text=99 usage=456.23 -> 456.23 )
- ./icapp -Ftl -D2 -H -K3 -k| file.csv "like prev., use '|' as separator
-
- - Text file: multiple numbers separated by non-digits (0..9,-,.) characters (ex. 134534,-45678,98788,4345, )
-
- ./icapp -Fc data.txt "text file, 32 bits integers (ex. 56789,3245,23,678 )
- ./icapp -Fcd data.txt "text file, 64 bits floting-point numbers (ex. 34.7689,5.20,45.789 )
-
- - Multiblocks of 32 bits binary file. (Example gov2 from [DocId data set](#DocId))<br />
- Block format: [n1: #of Ids][Id1] [Id2]...[IdN] [n2: #of Ids][Id1][Id2]...[IdN]...
-
- ./icbench -fS -r gov2.sorted
-
-
-##### - Intersections:
- 1 - Download Gov2 (or ClueWeb09) + query files (Ex. "1mq.txt") from [DocId data set](#DocId)<br />
- 8GB RAM required (16GB recommended for benchmarking "clueweb09" files).
-
- 2 - Create index file
-
-
- ./idxcr gov2.sorted .
-
-
- >*create inverted index file "gov2.sorted.i" in the current directory*
-
- 3 - Test intersections
-
-
- ./idxqry gov2.sorted.i 1mq.txt
-
-
- >*run queries in file "1mq.txt" over the index of gov2 file*
-
-##### - Parallel Query Processing:
- 1 - Create partitions
-
-
- ./idxseg gov2.sorted . -26m -s8
-
-
- >*create 8 (CPU hardware threads) partitions for a total of ~26 millions document ids*
-
- 2 - Create index file for each partition
-
-
- ./idxcr gov2.sorted.s*
-
-
- >*create inverted index file for all partitions "gov2.sorted.s00 - gov2.sorted.s07" in the current directory*
-
- 3 - Intersections:
-
- delete "idxqry.o" file and then type "make para" to compile "idxqry" w. multithreading
-
-
- ./idxqry gov2.sorted.s*.i 1mq.txt
-
- >*run queries in file "1mq.txt" over the index of all gov2 partitions "gov2.sorted.s00.i - gov2.sorted.s07.i".*
-
-### Function usage:
-See benchmark "icbench" program for "integer compression" usage examples.
-In general encoding/decoding functions are of the form:
-
- >**char *endptr = encode( unsigned *in, unsigned n, char *out, [unsigned start], [int b])**<br />
- endptr : set by encode to the next character in "out" after the encoded buffer<br />
- in : input integer array<br />
- n : number of elements<br />
- out : pointer to output buffer<br />
- b : number of bits. Only for bit packing functions<br />
- start : previous value. Only for integrated delta encoding functions
-
-
- >**char *endptr = decode( char *in, unsigned n, unsigned *out, [unsigned start], [int b])**<br />
- endptr : set by decode to the next character in "in" after the decoded buffer<br />
- in : pointer to input buffer<br />
- n : number of elements<br />
- out : output integer array<br />
- b : number of bits. Only for bit unpacking functions<br />
- start : previous value. Only for integrated delta decoding functions
-
- **Simple high level functions:**
- >**size_t compressed_size = encode( unsigned *in, size_t n, char *out)**<br />
- compressed_size : number of bytes written into compressed output buffer out<br />
-
- >**size_t compressed_size = decode( char *in, size_t n, unsigned *out)**<br />
- compressed_size : number of bytes read from compressed input buffer in<br />
-
-### Function syntax:
- - {vb | p4 | bit | vs}[n][d | d1 | f | fm | z ]{enc/dec | pack/unpack}[| 128V | 256V][8 | 16 | 32 | 64]:<br />
- vb: variable byte<br />
- p4: turbopfor<br />
- vs: variable simple<br />
- bit: bit packing<br />
- n : high level array functions for large arrays.
-
- '' : encoding for unsorted integer lists<br />
- 'd' : delta encoding for increasing integer lists (sorted w/ duplicate)<br />
- 'd1': delta encoding for strictly increasing integer lists (sorted unique)<br />
- 'f' : FOR encoding for sorted integer lists<br />
- 'z' : ZigZag encoding for unsorted integer lists<br />
-
- 'enc' or 'pack' : encode or bitpack<br />
- 'dec' or 'unpack': decode or bitunpack<br />
- 'NN' : integer size (8/16/32/64)<br />
-
-header files to use with documentation:<br />
-
-| c/c++ header file|Integer Compression functions| examples |
-|------------|-----------------------------|-----------------|
-|vint.h|variable byte| vbenc32/vbdec32 vbdenc32/vbddec32 vbzenc32/vbzdec32 |
-|vsimple.h|variable simple| vsenc64/vsdec64 |
-|vp4.h|TurboPFor| p4enc32/p4dec32 p4denc32/p4ddec32 p4zenc32/p4zdec32 |
-|bitpack.h|Bit Packing, For, +Direct Access| bitpack256v32/bitunpack256v32 bitforenc64/bitfordec64|
-|eliasfano.h|Elias Fano| efanoenc256v32/efanoc256v32 |
-
-Note: Some low level functions (like p4enc32) are limited to 128/256 (SSE/AVX2) integers per call.
-
-### Environment:
-###### OS/Compiler (64 bits):
-- Windows: MinGW-w64 makefile
-- Windows: Visual c++ (>=VS2008) - makefile.vs (for nmake)
-- Windows: Visual Studio project file - vs/vs2017 - Thanks to [PavelP](https://github.com/pps83)
-- Linux amd64: GNU GCC (>=4.6)
-- Linux amd64: Clang (>=3.2)
-- Linux arm64: 64 bits aarch64 ARMv8: gcc (>=6.3)
-- Linux arm64: 64 bits aarch64 ARMv8: clang
-- MaxOS: XCode (>=9)
-- MaxOS: Apple M1 (Clang)
-- PowerPC ppc64le (incl. SIMD): gcc (>=8.0)
-
-###### Multithreading:
-- All TurboPFor integer compression functions are thread safe
-
-### References:
-
-* [TurboPFor: an analysis](https://michael.stapelberg.ch/posts/2019-02-05-turbopfor-analysis/)
-
-* **Applications:**
- * [Debian Code Search](https://github.com/Debian/dcs/)</br>
- [Debian Code Search: positional index, TurboPFor-compressed](https://michael.stapelberg.ch/posts/2019-09-29-dcs-positional-turbopfor-index/)
- * [Graph500](https://github.com/julianromera/graph500)
- * [Small Polygon Compression](https://arxiv.org/abs/1509.05505) + [Poster](http://abhinavjauhri.me/publications/dcc_poster_2016.pdf) + [code](https://github.com/ajauhri/bignum_compression)
- * [Parallel Graph Analysis (Lecture 18)](http://www.cs.rpi.edu/~slotag/classes/FA16/) + [code](http://www.cs.rpi.edu/~slotag/classes/FA16/handson/lec18-comp2.cpp)
-
-* **Benchmark references:**
- * <a name="FastPFor"></a>[FastPFor](https://github.com/lemire/FastPFor) + [Simdcomp](https://github.com/lemire/simdcomp): SIMDPack FPF, Vbyte FPF, VarintG8IU, StreamVbyte, GroupSimple
- * <a name="OptPFD"></a><a name="Simple16"></a>[Optimized Pfor-delta compression code](http://jinruhe.com): OptPFD/OptP4, Simple16 (limited to 28 bits integers)
- * <a name="MaskedVByte"></a>[MaskedVByte](http://maskedvbyte.org/). See also: [Vectorized VByte Decoding](http://engineering.indeed.com/blog/2015/03/vectorized-vbyte-decoding-high-performance-vector-instructions/)
- * <a name="Streamvbyte"></a>[Streamvbyte](https://github.com/lemire/streamvbyte).
- * <a name="Simple-8b"></a>[Index Compression Using 64-Bit Words](http://people.eng.unimelb.edu.au/ammoffat/abstracts/am10spe.html): Simple-8b (speed optimized version tested)
- * <a name="libfor"></a>[libfor](https://github.com/cruppstahl/for)
- * <a name="QMX"></a>[Compression, SIMD, and Postings Lists](http://www.cs.otago.ac.nz/homepages/andrew/papers/) QMX integer compression from the "simple family"
- * <a name="lz4"></a>[lz4](https://github.com/Cyan4973/lz4). included w. block size 64K as indication. Tested after preprocessing w. delta+transpose
- * <a name="blosc"></a>[blosc](https://github.com/Blosc/c-blosc). blosc is like transpose/shuffle+lz77. Tested blosc+lz4 and blosclz incl. vectorizeed shuffle.<br>
- * <a name="DocId"></a>[Document identifier data set](http://lemire.me/data/integercompression2014.html)
-
-* **Integer compression publications:**
- * :green_book:[Evaluating Lightweight Integer Compression Algorithms in Column-Oriented In-Memory DBMS](http://www.adms-conf.org/2021-camera-ready/heinzl_adms21.pdf)
- * :green_book:[In Vacuo and In Situ Evaluation of SIMD Codecs (TurboPackV,TurboPFor/QMX)](http://dl.acm.org/citation.cfm?id=3015023) + [paper](http://www.cs.otago.ac.nz/homepages/andrew/papers/)
- * :green_book:[SIMD Compression and the Intersection of Sorted Integers](http://arxiv.org/abs/1401.6399)
- * :green_book:[Partitioned Elias-Fano Indexes](http://www.di.unipi.it/~ottavian/files/elias_fano_sigir14.pdf)
- * :green_book:[On Inverted Index Compression for Search Engine Efficiency](http://www.dcs.gla.ac.uk/~craigm/publications/catena14compression.pdf)
- * :green_book:[Google's Group Varint Encoding](http://static.googleusercontent.com/media/research.google.com/de//people/jeff/WSDM09-keynote.pdf)
- * :green_book:[Integer Compression tweets](https://twitter.com/search?q=%23integercompression&src=typd)
- * :green_book:[Efficient Compression of Scientific Floating-Point Data and An Application in Structural Analysis](https://www.jstage.jst.go.jp/article/jsces/2017/0/2017_20170002/_article)
- * :green_book:[SPDP is a compression/decompression algorithm for binary IEEE 754 32/64 bits floating-point data](http://cs.txstate.edu/~burtscher/research/SPDPcompressor/)<br />
- :green_book:[ SPDP - An Automatically Synthesized Lossless Compression Algorithm for Floating-Point Data](http://cs.txstate.edu/~mb92/papers/dcc18.pdf) + [DCC 2018](http://www.cs.brandeis.edu//~dcc/Programs/Program2018.pdf)
-
-Last update: 13 Nov 2021
-
-## APPENDIX: icbench Integer Compression Benchmark
-
-##### TurboPFor + external libraries
-<pre>
-TurboPFor https://github.com/powturbo/TurboPFor
-FastPFor (FP) https://github.com/lemire/FastPFor
-lz4 https://github.com/Cyan4973/lz4
-LittleIntPacker (LI) https://github.com/lemire/LittleIntPacker
-MaskedVbyte http://maskedvbyte.org
-Polycom (PC) https://github.com/encode84/bcm
-simdcomp (SC) https://github.com/lemire/simdcomp
-Simple-8b optimized https://github.com/powturbo/TurboPFor
-Streamvbyte https://github.com/lemire/streamvbyte
-VarintG8IU https://github.com/lemire/FastPFor
-</pre>
-
-##### Functions integrated into 'icbench' for benchmarking
-<pre>
-Codec group:
-TURBOPFOR TurboPFor library TurboPFor256V/TurboPack256V/TurboPFor256N/TurboPFor/TurboPackV/TurboVByte/TurboPack/TurboForDA/EliasFano/VSimple/TurboPForN/TurboPackN/TurboPForDI
-DEFAULT Default TurboPFor/TurboPackV/TurboVByte/TurboPack/TurboFor/TurboPForN/TurboPackN/TurboPForDI/TurboPFor256V/TurboPack256V/TurboPFor256N
-BENCH Benchmark TurboPFor/TurboPackV/TurboVByte/TurboPack/QMX/FP.SimdFastPfor/FP.SimdOptPFor/MaskedVbyte/StreamVbyte
-EFFICIENT Efficient TurboPFor/vsimple/turbovbyte
-TRANSFORM transpose/shufle,delta,zigzag tpbyte4s/tpbyte,4/tpnibble,4/ZigZag_32/Delta_32/BitShuffle,4
-BITPACK Bit Packing TurboPack256V/TurboPackV/TurboPackH/TurboPack/SC.SimdPack128/SC.SimdPack256
-VBYTE Variable byte TurboVByte/FP.VByte/PC.Vbyte/VarintG8IU/MaskedVbyte/StreamVbyte
-SIMPLE Simple Family simple8b/simple16/vsimple/qmx
-LZ4 lz4+bitshufle/transpose 4,8 lz4_bitshufle/lz4_tp4/lz4_tp8
-LI Little Integer LI_Pack/LI_TurboPack/LI_SuperPack/LI_HorPack
-
-
-Function Description level
-
--------- ----------- -----
-TurboPFor PFor (SSE2)
-TurboPForN PFor (SSE2) large blocks
-TurboPFor256 PFor (AVX2)
-TurboPFor256N PFor (AVX2) large blocks
-TurboPForDA PFor direct access
-TurboPForDI PFord min
-TurboPForZZ PFor zigzag of delta
-TurboFor FOR
-TurboForV FOR (SIMD)
-TurboFor256V FOR (AVX2)
-TurboForDA FOR direct access
-TurboPackDA Bit packing direct access
-TurboPack Bit packing (scalar)
-TurboPackN Bit packing (scalar) large blocks
-TurboPackV Bit packing (SSE2 Vertical)
-TurboPackH Bit packing (SSE2 Horizontal)
-TurboPackVN Bit packing (SSE2 large block)
-TurboPack256V Bit packing (AVX2 Vertical)
-TurboPack256N Bit packing (AVX2 large block)
-TurboVByte Variable byte (scalar)
-VSimple Variable simple (scalar)
-EliasFano Elias fano (scalar)
-EliasFanoV Eliasfano (SSE2)
-EliasFano256V Elias fano (AVX2)
-memcpy memcpy
-copy Integer copy
-tpbyte4s Byte Transpose (scalar)
-tpbyte Byte transpose (simd) 2,4,8
-tpnibble Nibble transpose (simd) 2,4,8
-ZigZag32 ZigZag encoding (sse2)
-Delta32 Delta encoding (sse2)
-DDelta32 Delta of delta encoding (sse2)
-Xor32 Xor encoding (sse2)
-FP_PREV64 Floating point PFOR
-FP_FCM64 Floating point PFOR (FCM)
-FP_DFCM64 Floating point PFOR (DFCM)
-TurboPFor64 PFOR 64
-TurboPFor64V PFOR 64
-Simple8b 64 bits Simple family (instable)
-PC_Simple16 Simple 16. limited to 28 bits
-PC_OptPFD OptPFD. limited to 28 bits
-PC_Vbyte Variable byte
-PC_Rice Rice coding (instable)
-VarintG8IU Variable byte SIMD
-MaskedVbyte Variable byte SIMD
-StreamVbyte Variable byte SIMD
-FP_FastPFor PFor scalar (inefficient for small blocks)
-FP_SimdFastPFor PFor SIMD (inefficient for small blocks)
-FP_OptPFor OptPFor scalar
-FP_SIMDOptPFor OptPFor SIMD
-FP_VByte Variable byte
-FP_Simple8bRLE Simple-8b + rle
-FP_GROUPSIMPLE Group Simple
-SC_SIMDPack128 Bit packing (SSE4.1)
-SC_SIMDPack256 Bit packing (SSE4.1)
-SC_For For (SSE4.1)
-SC_ForDA For direct access (SSE4.1)
-LibFor_For For
-LibFor_ForDA For direct access
-LI_Pack Bit packing (scalar)
-LI_TurboPack Bit packing (scalar)
-LI_SuperPack Bit packing (scalar)
-LI_HorPack Bit packing (sse4.1 horizontal)
-LI_BMIPack256 Bit packing (avx2)
-lz4 lz4
-lz4_bit Bitshuffle + [delta]+lz4 2,4,8
-lz4_nibble TurboPFor's [delta]+nibble transpose + lz4 2,4,8
-lz4_bitxor Bitshuffle + [xor]+lz4 2,4,8
-lz4_nibblexor TurboPFor's [xor]+nibble transpose + lz4 2,4,8
-lz4_byte TurboPFor's [delta]+byte transpose + lz4 2,4,8
-BitShuffle Bit shuffle (simd) 2,4,8
-</pre>
-
diff --git a/src/ext/for/bitpack.c b/src/ext/for/bitpack.c
deleted file mode 100644
index 666b1030e51..00000000000
--- a/src/ext/for/bitpack.c
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// "Integer Compression" bit packing
-
-#include <stdio.h>
-#define BITUTIL_IN
-#define VINT_IN
-#include "conf.h"
-#include "bitutil.h"
-#include "vint.h"
-#include "bitpack.h"
-#define PAD8(_x_) ( (((_x_)+8-1)/8) )
-
- #ifdef __ARM_NEON
-#define PREFETCH(_ip_,_rw_)
- #else
-#define PREFETCH(_ip_,_rw_) __builtin_prefetch(_ip_,_rw_)
- #endif
-
-#pragma warning( disable : 4005)
-#pragma warning( disable : 4090)
-#pragma warning( disable : 4068)
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunsequenced"
-
-#if !defined(SSE2_ON) && !defined(AVX2_ON) //----------------------------------- Plain -----------------------------------------------------------------------
-typedef unsigned char *(*BITPACK_F8)( uint8_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D8)( uint8_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint8_t start);
-typedef unsigned char *(*BITPACK_F16)(uint16_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D16)(uint16_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint16_t start);
-typedef unsigned char *(*BITPACK_F32)(uint32_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D32)(uint32_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint32_t start);
-typedef unsigned char *(*BITPACK_F64)(uint64_t *__restrict out, unsigned n, const unsigned char *__restrict in);
-typedef unsigned char *(*BITPACK_D64)(uint64_t *__restrict out, unsigned n, const unsigned char *__restrict in, uint64_t start);
-
-#if 1 //def _MSC_VER
-#define VX (v=x)
-#define V x
-#else
-#define VX v
-#define V v
-#endif
-
-#if 0
-#define IP0(_ip_,_x_) *_ip_
-#define IP( _ip_,_x_) *_ip_++
-#define IPI(_ip_)
-#else
-#define IP0(_ip_,_x_) _ip_[_x_]
-#define IP( _ip_,_x_) _ip_[_x_]
-#define IPI(_ip_) _ip_ += 32
-#endif
-
-#define IP9(_ip_,_x_, _parm_)
-#define IPW(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_))
-
-#define IPV(_ip_,_x_) IP(_ip_,_x_)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define DELTA
-
-#define IP9(_ip_,_x_, _parm_) V = IP0(_ip_,_x_) - start; start = IP(_ip_,_x_)
-#define IPV(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start)
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitdpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IP9(_ip_,_x_, _parm_)
-#define IPV(_ip_,_x_) IP(_ip_,_x_) - start
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitfpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IP9( _ip_,_x_, _parm_) V = IP0(_ip_,_x_) - start - 1; start = IP(_ip_,_x_)
-#define IPV( _ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start - 1)
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitd1pack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-/*#define IP9( _ip_,_x_, _parm_) v = IP(_ip_,_x_) - start - mdelta; start = IP(_ip_,_x_)
-#define IPV( _ip_,_x_) v
-#define IPX(_ip_,_x_) (v = IP(_ip_,_x_) - start - mdelta)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitepack
-#include "bitpack_.h"*/
-
-#define IP9(_ip_,_x_, _parm_) V = TEMPLATE2(zigzagenc, USIZE)(IP(_ip_,_x_) - start); start = IP(_ip_,_x_)
-#define IPV(_ip_,_x_) VX
-#define IPX(_ip_,_x_) (V = TEMPLATE2(zigzagenc, USIZE)(IP(_ip_,_x_) - start))
-#define IP16(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP32(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define IP64(_ip_,_x_, _parm_) start = IP(_ip_,_x_)
-#define _BITPACK_ bitzpack
-#include "bitpack_.h"
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define IPI(_ip_) _ip_ += 32; start += 32
-#define IP9(_ip_,_x_, _parm_)
-#define IPV(_ip_,_x_) (IP(_ip_,_x_) - start - (_x_) - 1)
-#define IPX(_ip_,_x_) (V = IP(_ip_,_x_) - start - (_x_) - 1)
-#define IP16(_ip_,_x_, _parm_)
-#define IP32(_ip_,_x_, _parm_)
-#define IP64(_ip_,_x_, _parm_)
-#define _BITPACK_ bitf1pack
-#include "bitpack_.h"
-#undef IPI
-#undef IP9
-#undef IPV
-#undef IPX
-#undef IP16
-#undef IP32
-#undef IP64
-
-#define BITNPACK(in, n, out, _csize_, _usize_) { unsigned char *op = out;\
- for(ip = in, in += n; ip < in;) { \
- TEMPLATE3(uint, _usize_, _t) o,x;\
- unsigned iplen = in - ip,b; \
- if(iplen > _csize_) iplen = _csize_; \
- PREFETCH(ip+512,0);\
- o = TEMPLATE2(bit,_usize_)(ip, iplen, &x); b = TEMPLATE2(bsr,_usize_)(o);\
- *op++ = b; op = TEMPLATE2(bitpacka, _usize_)[b](ip, iplen, op);\
- ip += iplen;\
- }\
- return op - out;\
-}
-
-#define BITNDPACK(in, n, out, _csize_, _usize_, _bitd_, _bitpacka_) { if(!n) return 0;\
- unsigned char *op = out; \
- TEMPLATE3(uint, _usize_, _t) o,x;\
- start = *in++; \
- TEMPLATE2(vbxput, _usize_)(op, start);\
- for(n--,ip = in; ip != in + (n&~(_csize_-1)); ) { unsigned b; PREFETCH(ip+512,0);\
- o = TEMPLATE2(_bitd_, _usize_)(ip, _csize_, &x, start); b = TEMPLATE2(bsr,_usize_)(o); *op++ = b; op = TEMPLATE2(_bitpacka_,_usize_)[b](ip, _csize_, op, start); ip += _csize_; start = ip[-1];\
- }\
- if(n&=(_csize_-1)) { unsigned b;\
- o = TEMPLATE2(_bitd_, _usize_)(ip, n, &x, start); b = TEMPLATE2(bsr,_usize_)(o); *op++ = b; op = TEMPLATE2(_bitpacka_,_usize_)[b](ip, n, op, start);\
- }\
- return op - out;\
-}
-
-size_t bitnpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNPACK(in, n, out, 128, 8); }
-size_t bitnpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNPACK(in, n, out, 128, 16); }
-size_t bitnpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNPACK(in, n, out, 128, 32); }
-size_t bitnpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNPACK(in, n, out, 128, 64); }
-
-size_t bitndpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitd, bitdpacka); }
-size_t bitndpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitd, bitdpacka); }
-size_t bitndpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitd, bitdpacka); }
-size_t bitndpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitd, bitdpacka); }
-
-size_t bitnd1pack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitd1, bitd1packa); }
-size_t bitnd1pack16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitd1, bitd1packa); }
-size_t bitnd1pack32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitd1, bitd1packa); }
-size_t bitnd1pack64(uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitd1, bitd1packa); }
-
-size_t bitnzpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitz, bitzpacka); }
-size_t bitnzpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitz, bitzpacka); }
-size_t bitnzpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitz, bitzpacka); }
-size_t bitnzpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitz, bitzpacka); }
-
-size_t bitnfpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out) { uint8_t *ip,start; BITNDPACK(in, n, out, 128, 8, bitf, bitfpacka); }
-size_t bitnfpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; BITNDPACK(in, n, out, 128, 16, bitf, bitfpacka); }
-size_t bitnfpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; BITNDPACK(in, n, out, 128, 32, bitf, bitfpacka); }
-size_t bitnfpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip,start; BITNDPACK(in, n, out, 128, 64, bitf, bitfpacka); }
-
-#else //--------------------------------------- SIMD ----------------------------------------------------------------------------------------------
-
-#define _BITNPACKV(in, n, out, _csize_, _usize_, _bitpackv_) {\
- unsigned char *op = out; TEMPLATE3(uint, _usize_, _t) _o,_x;\
- for(ip = in; ip != in + (n&~(_csize_-1)); ip += _csize_) { PREFETCH(ip+512,0);\
- unsigned _b; _o = TEMPLATE2(bit,_usize_)(ip, _csize_, &_x); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpackv_, _usize_)(ip, _csize_, op, _b);\
- } if(n&=(_csize_-1)) { unsigned _b; _o = TEMPLATE2(bit,_usize_)(ip, n, &_x); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(bitpack, _usize_)(ip, n, op, _b); }\
- return op - out;\
-}
-
-#define _BITNDPACKV(in, n, out, _csize_, _usize_, _bitdv_, _bitpackv_, _bitd_, _bitpack_) { if(!n) return 0;\
- unsigned char *op = out; TEMPLATE3(uint, _usize_, _t) _o,_x;\
- start = *in++; \
- TEMPLATE2(vbxput, _usize_)(op, start);\
- for(n--,ip = in; ip != in + (n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned _b; _o = TEMPLATE2(_bitdv_, _usize_)(ip, _csize_, &_x, start); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpackv_, _usize_)(ip, _csize_, op, start, _b); ip += _csize_; start = ip[-1];\
- } if(n&=(_csize_-1)) { unsigned _b; _o = TEMPLATE2(_bitd_, _usize_)(ip, n, &_x, start); _b = TEMPLATE2(bsr,_usize_)(_o); *op++ = _b; op = TEMPLATE2(_bitpack_, _usize_)(ip, n, op, start, _b); }\
- return op - out;\
-}
-
- #ifdef __AVX2__
-#include <immintrin.h>
-#include "bitpack_.h"
-
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define PAD8(__x) (((__x)+8-1)/8)
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define VI32(ip, i, iv, parm)
-#define IP32(ip, i, iv) _mm256_loadu_si256(ip++)
-
-unsigned char *bitpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(256*b); BITPACK256V32(in, b, out, 0); return pout; }
-#undef VI32
-#undef IP32
-
-
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm256_sub_epi32(_mm256_loadu_si256(_ip_++),sv)
-#define IP32(_ip_, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitfpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm256_sub_epi32(_mm256_loadu_si256(_ip_++),_sv_); _sv_ = _mm256_add_epi32(_sv_,cv);
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitf1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set_epi32(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1), cv = _mm256_set1_epi32(8);
- BITPACK256V32(in, b, out, sv); return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = mm256_delta_epi32(v,_sv_); _sv_ = v
-#define IP32(ip, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitdpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v,sv = _mm256_set1_epi32(start);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = _mm256_sub_epi32(mm256_delta_epi32(v,_sv_),cv); _sv_ = v
-unsigned char *bitd1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start), cv = _mm256_set1_epi32(1);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm256_loadu_si256(_ip_++); _iv_ = mm256_delta_epi32(v,_sv_); _sv_ = v; _iv_ = mm256_zzage_epi32(_iv_)
-unsigned char *bitzpack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(256*b);
- __m256i v, sv = _mm256_set1_epi32(start), cv = _mm256_set1_epi32(1);
- BITPACK256V32(in, b, out, sv);
- return pout;
-}
-
-size_t bitnpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 256, 32, bitpack256v); }
-size_t bitndpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitd, bitdpack256v, bitd, bitdpack); }
-size_t bitnd1pack256v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitd1, bitd1pack256v,bitd1, bitd1pack); }
-size_t bitnzpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitz, bitzpack256v, bitz, bitzpack); }
-size_t bitnfpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 256, 32, bitf, bitfpack256v, bitf, bitfpack); }
-
- #elif defined(__SSE2__) || defined(__ARM_NEON) //----------------------------- SSE ---------------------------------------------------------------
-#define OPPE(__op)
-#define IPPE(__op)
-
-#define VI16(ip, i, iv, parm)
-#define VI32(ip, i, iv, parm)
-#define IP16(_ip_, i, iv) _mm_loadu_si128(_ip_++)
-#define IP32(_ip_, i, iv) _mm_loadu_si128(_ip_++)
-#include "bitpack_.h"
-unsigned char *bitpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(128*b); BITPACK128V16(in, b, out, 0); return pout; }
-unsigned char *bitpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *pout = out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return pout; }
-unsigned char *bitpack256w32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) { unsigned char *_out=out; unsigned *_in=in;
-BITPACK128V32(in, b, out, 0); in = _in+128; out = _out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return _out+PAD8(256*b); }
-
-#ifdef __ARM_NEON
-//#define IP32(_ip_, i, iv) _mm_or_si128(_mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(3, 1, 2, 0)), _mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(2, 0, 3, 1)) )
-#define IP32(_ip_, _i_, _iv_) _mm_or_si128(mm_shuffle_3120_epi32(_mm_loadu_si128(_ip_++) ), mm_shuffle_2031_epi32(_mm_loadu_si128(_ip++) ) ) // optimized shuffle
-#else
-#define IP32(_ip_, i, iv) _mm_or_si128(_mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(2, 0, 3, 1)), _mm_shuffle_epi32( _mm_loadu_si128(_ip_++),_MM_SHUFFLE(3, 1, 2, 0)) )
-#endif
-#include "bitpack_.h"
-unsigned char *bitpack128v64(uint64_t *__restrict in, unsigned n, unsigned char *__restrict out, unsigned b) {
- if(b<=32) { unsigned char *pout = out+PAD8(128*b); BITPACK128V32(in, b, out, 0); return pout; } else return bitpack64(in,n,out,b);
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi16(v,_sv_); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi32(v,_sv_); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-#include "bitpack_.h"
-unsigned char *bitdpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v,sv = _mm_set1_epi16(start); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitdpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v,sv = _mm_set1_epi32(start); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_)
-#define VI32(_ip_, _i_, _iv_, _sv_)
-#define IP16(_ip_, i, _iv_) _mm_sub_epi16(_mm_loadu_si128(_ip_++),sv)
-#define IP32(_ip_, i, _iv_) _mm_sub_epi32(_mm_loadu_si128(_ip_++),sv)
-#include "bitpack_.h"
-unsigned char *bitfpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitfpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi16(mm_delta_epi16(v,_sv_),cv); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi32(mm_delta_epi32(v,_sv_),cv); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitd1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(1), v; BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitd1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(1); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi16(SUBI16x8(v,_sv_),cv); _sv_ = v
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = _mm_sub_epi32(SUBI32x4(v,_sv_),cv); _sv_ = v
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bits1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(8); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bits1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(4); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) _iv_ = _mm_sub_epi16(_mm_loadu_si128(_ip_++),_sv_); _sv_ = _mm_add_epi16(_sv_,cv);
-#define VI32(_ip_, _i_, _iv_, _sv_) _iv_ = _mm_sub_epi32(_mm_loadu_si128(_ip_++),_sv_); _sv_ = _mm_add_epi32(_sv_,cv);
-#define IP16(ip, i, _iv_) _iv_
-#define IP32(ip, i, _iv_) _iv_
-unsigned char *bitf1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set_epi16(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1), cv = _mm_set1_epi16(8); BITPACK128V16(in, b, out, sv); return pout;
-}
-unsigned char *bitf1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set_epi32( start+4,start+3,start+2,start+1), cv = _mm_set1_epi32(4); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-#define VI16(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi16(v,_sv_); _sv_ = v; _iv_ = mm_zzage_epi16(_iv_)
-unsigned char *bitzpack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(1); BITPACK128V16(in, b, out, sv); return pout;
-}
-#define VI32(_ip_, _i_, _iv_, _sv_) v = _mm_loadu_si128(_ip_++); _iv_ = mm_delta_epi32(v,_sv_); _sv_ = v; _iv_ = mm_zzage_epi32(_iv_)
-unsigned char *bitzpack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b) { unsigned char *pout = out+PAD8(128*b);
- __m128i v, sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(1); BITPACK128V32(in, b, out, sv); return pout;
-}
-
-size_t bitnpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip; _BITNPACKV( in, n, out, 128, 16, bitpack128v); }
-size_t bitnpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 128, 32, bitpack128v); }
-size_t bitnpack128v64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out) { uint64_t *ip; _BITNPACKV( in, n, out, 128, 64, bitpack128v); }
-size_t bitnpack256w32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip; _BITNPACKV( in, n, out, 256, 32, bitpack256w); }
-
-size_t bitndpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitd, bitdpack128v, bitd, bitdpack); }
-size_t bitndpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitd, bitdpack128v, bitd, bitdpack); }
-
-size_t bitnd1pack128v16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitd1, bitd1pack128v, bitd1, bitd1pack); }
-size_t bitnd1pack128v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitd1, bitd1pack128v, bitd1, bitd1pack); }
-
-size_t bitns1pack128v16(uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bits128v, bits1pack128v, bitd1, bitd1pack); }
-size_t bitns1pack128v32(uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bits128v, bits1pack128v, bitd1, bitd1pack); }
-
-size_t bitnzpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitz, bitzpack128v, bitz, bitzpack); }
-size_t bitnzpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitz, bitzpack128v, bitz, bitzpack); }
-
-size_t bitnfpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out) { uint16_t *ip,start; _BITNDPACKV(in, n, out, 128, 16, bitf, bitfpack128v, bitf, bitfpack); }
-size_t bitnfpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out) { uint32_t *ip,start; _BITNDPACKV(in, n, out, 128, 32, bitf, bitfpack128v, bitf, bitfpack); }
- #endif // SSE
-#endif // Plain
-
-#pragma clang diagnostic pop
diff --git a/src/ext/for/bitpack.h b/src/ext/for/bitpack.h
deleted file mode 100644
index b0b9e02275a..00000000000
--- a/src/ext/for/bitpack.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// bitpack.h - "Integer Compression" Binary Packing header file
-#ifndef BITPACK_H_
-#define BITPACK_H_
-#if defined(_MSC_VER) && _MSC_VER < 1600
-#include "vs/stdint.h"
-#else
-#include <stdint.h>
-#endif
-#include <stddef.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-//******************** Bit Packing High Level API - n unlimited ***************************************************
-size_t bitnpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack128v64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitndpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitndpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnd1pack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnd1pack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnzpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnzpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnfpack8( uint8_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack64( uint64_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack128v16( uint16_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack128v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-size_t bitnfpack256v32( uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
-
-size_t bitnunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnunpack128v64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitndunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitndunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitndunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitndunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitndunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitndunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitndunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnd1unpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnd1unpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnd1unpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnd1unpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnd1unpack128v16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnd1unpack128v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnd1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnzunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnzunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnzunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnzunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnzunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnzunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnzunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-
-size_t bitnfunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out);
-size_t bitnfunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnfunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnfunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out);
-size_t bitnfunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out);
-size_t bitnfunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-size_t bitnfunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
-//******** Bit Packing Low level API ****************************************************************
-// bipackNN: Pack array with n unsigned (NN bits in[n]) values to the buffer out using nbits per value. Return value = end of compressed buffer out
-unsigned char *bitpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b);
-
-// delta bit packing
-unsigned char *bitdpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitdpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitdpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitdpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-unsigned char *bitd1pack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitd1pack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitd1pack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitd1pack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-// FOR bit packing : sorted integer array
-unsigned char *bitfpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitfpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitfpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitfpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-unsigned char *bitf1pack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitf1pack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitf1pack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitf1pack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-// zigzag : unsorted integer array
-unsigned char *bitzpack8( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitzpack16( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitzpack32( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitzpack64( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out, uint64_t start, unsigned b);
-
-//-------------------------------------- SIMD ------------------------------------------------------------------------------------------
-// Pack array with 128 unsigned (32 bits in[n]) values to the buffer out using nbits per value. Return value = end of compressed buffer out
-unsigned char *bitpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitd1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitfpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitf1pack128v16(unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitzpack128v16( unsigned short *__restrict in, unsigned n, unsigned char *__restrict out, unsigned short start, unsigned b);
-
-unsigned char *bitpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1pack128v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitzpack128v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-
-//unsigned char *bitpack256w32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitpack128v64( uint64_t *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-
-unsigned char *bitpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out , unsigned b);
-unsigned char *bitdpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1pack256v32(unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-unsigned char *bitzpack256v32( unsigned *__restrict in, unsigned n, unsigned char *__restrict out, unsigned start, unsigned b);
-
-//********************************** Bit Packing : Unpack ****************************************************************
-
-// ---------------- Unpack a b-bits packed integer array -------------------------------------------------------------------------------
-// unpack a bitpacked integer array. Return value = end of packed buffer in
-unsigned char *bitunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, unsigned b);
-unsigned char *bitunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, unsigned b);
-unsigned char *bitunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, unsigned b);
-unsigned char *bitunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b);
-
-// ---------------- Direct Access to a single packed integer array entry --------------------------------------------------------------
- #ifdef TURBOPFOR_DAC
- #ifdef __AVX2__
-#include <immintrin.h>
-#define bzhi64(_u_, _b_) _bzhi_u64(_u_, _b_)
-#define bzhi32(_u_, _b_) _bzhi_u32(_u_, _b_)
- #else
-#define bzhi64(_u_, _b_) ((_u_) & ((1ull<<(_b_))-1))
-#define bzhi32(_u_, _b_) ((_u_) & ((1u <<(_b_))-1))
- #endif
-
-#include "conf.h"
-
-static ALWAYS_INLINE unsigned bitgetx32(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-//static ALWAYS_INLINE unsigned bitgetx32(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx;
- //return (ctou64((uint32_t *)in+(bidx>>5)) << 32+(bidx&0x1f)) >> (64-b);
-// return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-static ALWAYS_INLINE unsigned _bitgetx32(const unsigned char *__restrict in, uint64_t bidx, unsigned b) { return bzhi64( ctou64((uint32_t *)in+(bidx>>5)) >> (bidx&0x1f), b ); }
-
-// like bitgetx32 but for 16 bits integer array
-static ALWAYS_INLINE unsigned bitgetx8( const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi32( ctou16((uint16_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned _bitgetx8( const unsigned char *__restrict in, unsigned bidx, unsigned b) { return bzhi32( ctou16((uint16_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned bitgetx16(const unsigned char *__restrict in, unsigned idx, unsigned b) { unsigned bidx = b*idx; return bzhi32( ctou32((uint32_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-static ALWAYS_INLINE unsigned _bitgetx16(const unsigned char *__restrict in, unsigned bidx, unsigned b) { return bzhi32( ctou32((uint32_t *)in+(bidx>>4)) >> (bidx& 0xf), b ); }
-
-// Set a single value with index "idx"
-static ALWAYS_INLINE void bitsetx16(const unsigned char *__restrict in, unsigned idx, unsigned v, unsigned b) { unsigned bidx = b*idx; unsigned *p = (unsigned *) in+(bidx>>4) ; *p = ( *p & ~(((1u <<b)-1) << (bidx& 0xf)) ) | v<<(bidx& 0xf);}
-static ALWAYS_INLINE void bitsetx32(const unsigned char *__restrict in, unsigned idx, unsigned v, unsigned b) { unsigned bidx = b*idx; unsigned long long *p = (unsigned long long *)((unsigned *)in+(bidx>>5)); *p = ( *p & ~(((1ull<<b)-1) << (bidx&0x1f)) ) | (unsigned long long)v<<(bidx&0x1f);}
- #endif
-// ---------------- DFOR : integrated bitpacking, for delta packed SORTED array (Ex. DocId in inverted index) -------------------------------
-// start <= out[0] <= out[1] <= ... <= out[n-2] <= out[n-1] <= (1<<N)-1 N=8,16,32 or 64
-// out[0] = start + in[0]; out[1] = out[0] + in[1]; ... ; out[i] = out[i-1] + in[i]
-unsigned char *bitdunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitdunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitdunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitdunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// start < out[0] < out[1] < ... < out[n-2] < out[n-1] < (1<<N)-1, N=8,16,32 or 64
-// out[0] = start + in[0] + 1; out[1] = out[0] + in[1] + 1; ... ; out[i] = out[i-1] + in[i] + 1
-unsigned char *bitd1unpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitd1unpack16(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitd1unpack32(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitd1unpack64(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- ZigZag : integrated bitpacking, for zigzag packed unsorted
-unsigned char *bitzunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitzunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitzunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitzunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- For : Direct Access for packed SORTED array --------------------------------------------
-// out[i] = start + in[i] + i
-unsigned char *bitfunpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitfunpack16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitfunpack32( const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitfunpack64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// out[i] = start + in[i] + i + 1
-unsigned char *bitf1unpack8( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start, unsigned b);
-unsigned char *bitf1unpack16(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b);
-unsigned char *bitf1unpack32(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start, unsigned b);
-unsigned char *bitf1unpack64(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b);
-
-// ---------------- SIMD : unpack a SIMD bit packed integer array -------------------------------------------------------------------------------
-// SIMD unpack a 128/256 bitpacked integer array. Return value = end of packed buffer in
-unsigned char *bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b);
-unsigned char *bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitdunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitd1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitfunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-unsigned char *bitf1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b);
-
-unsigned char *bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-unsigned char *bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b);
-
-unsigned char *bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitfunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitf1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-unsigned char *bitunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b);
-unsigned char *bitzunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitdunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-unsigned char *bitd1unpack128h32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b);
-
-// internal TurboPFor functions: masked unpack
-unsigned char *_bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128v16(const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack128h32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack128h32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b, uint32_t *__restrict pex, unsigned char *bb);
-
-unsigned char *_bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitd1unpack256v32(const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-unsigned char *_bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb);
-#ifdef __cplusplus
-}
-#endif
-#endif
-
diff --git a/src/ext/for/bitpack_.h b/src/ext/for/bitpack_.h
deleted file mode 100644
index d20cf9f0fea..00000000000
--- a/src/ext/for/bitpack_.h
+++ /dev/null
@@ -1,5190 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2017
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// TurboPFor Integer Compression: Bit Packing
-#ifdef IPI
-#define BITBLK64_1(ip, i, op, parm) { uint32_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint32_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint32_t)IPV(ip, i*32+ 1) << 1;\
- IP9(ip, i*32+ 2, parm); w |= (uint32_t)IPV(ip, i*32+ 2) << 2;\
- IP9(ip, i*32+ 3, parm); w |= (uint32_t)IPV(ip, i*32+ 3) << 3;\
- IP9(ip, i*32+ 4, parm); w |= (uint32_t)IPV(ip, i*32+ 4) << 4;\
- IP9(ip, i*32+ 5, parm); w |= (uint32_t)IPV(ip, i*32+ 5) << 5;\
- IP9(ip, i*32+ 6, parm); w |= (uint32_t)IPV(ip, i*32+ 6) << 6;\
- IP9(ip, i*32+ 7, parm); w |= (uint32_t)IPV(ip, i*32+ 7) << 7;\
- IP9(ip, i*32+ 8, parm); w |= (uint32_t)IPV(ip, i*32+ 8) << 8;\
- IP9(ip, i*32+ 9, parm); w |= (uint32_t)IPV(ip, i*32+ 9) << 9;\
- IP9(ip, i*32+10, parm); w |= (uint32_t)IPV(ip, i*32+10) << 10;\
- IP9(ip, i*32+11, parm); w |= (uint32_t)IPV(ip, i*32+11) << 11;\
- IP9(ip, i*32+12, parm); w |= (uint32_t)IPV(ip, i*32+12) << 12;\
- IP9(ip, i*32+13, parm); w |= (uint32_t)IPV(ip, i*32+13) << 13;\
- IP9(ip, i*32+14, parm); w |= (uint32_t)IPV(ip, i*32+14) << 14;\
- IP9(ip, i*32+15, parm); w |= (uint32_t)IPV(ip, i*32+15) << 15;\
- IP9(ip, i*32+16, parm); w |= (uint32_t)IPV(ip, i*32+16) << 16;\
- IP9(ip, i*32+17, parm); w |= (uint32_t)IPV(ip, i*32+17) << 17;\
- IP9(ip, i*32+18, parm); w |= (uint32_t)IPV(ip, i*32+18) << 18;\
- IP9(ip, i*32+19, parm); w |= (uint32_t)IPV(ip, i*32+19) << 19;\
- IP9(ip, i*32+20, parm); w |= (uint32_t)IPV(ip, i*32+20) << 20;\
- IP9(ip, i*32+21, parm); w |= (uint32_t)IPV(ip, i*32+21) << 21;\
- IP9(ip, i*32+22, parm); w |= (uint32_t)IPV(ip, i*32+22) << 22;\
- IP9(ip, i*32+23, parm); w |= (uint32_t)IPV(ip, i*32+23) << 23;\
- IP9(ip, i*32+24, parm); w |= (uint32_t)IPV(ip, i*32+24) << 24;\
- IP9(ip, i*32+25, parm); w |= (uint32_t)IPV(ip, i*32+25) << 25;\
- IP9(ip, i*32+26, parm); w |= (uint32_t)IPV(ip, i*32+26) << 26;\
- IP9(ip, i*32+27, parm); w |= (uint32_t)IPV(ip, i*32+27) << 27;\
- IP9(ip, i*32+28, parm); w |= (uint32_t)IPV(ip, i*32+28) << 28;\
- IP9(ip, i*32+29, parm); w |= (uint32_t)IPV(ip, i*32+29) << 29;\
- IP9(ip, i*32+30, parm); w |= (uint32_t)IPV(ip, i*32+30) << 30;\
- IP9(ip, i*32+31, parm); w |= (uint32_t)IPV(ip, i*32+31) << 31;*((uint32_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_1(ip, op, parm) { \
- BITBLK64_1(ip, 0, op, parm); IPI(ip); op += 1*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_2(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint64_t)IPV(ip, i*32+ 1) << 2;\
- IP9(ip, i*32+ 2, parm); w |= (uint64_t)IPV(ip, i*32+ 2) << 4;\
- IP9(ip, i*32+ 3, parm); w |= (uint64_t)IPV(ip, i*32+ 3) << 6;\
- IP9(ip, i*32+ 4, parm); w |= (uint64_t)IPV(ip, i*32+ 4) << 8;\
- IP9(ip, i*32+ 5, parm); w |= (uint64_t)IPV(ip, i*32+ 5) << 10;\
- IP9(ip, i*32+ 6, parm); w |= (uint64_t)IPV(ip, i*32+ 6) << 12;\
- IP9(ip, i*32+ 7, parm); w |= (uint64_t)IPV(ip, i*32+ 7) << 14;\
- IP9(ip, i*32+ 8, parm); w |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); w |= (uint64_t)IPV(ip, i*32+ 9) << 18;\
- IP9(ip, i*32+10, parm); w |= (uint64_t)IPV(ip, i*32+10) << 20;\
- IP9(ip, i*32+11, parm); w |= (uint64_t)IPV(ip, i*32+11) << 22;\
- IP9(ip, i*32+12, parm); w |= (uint64_t)IPV(ip, i*32+12) << 24;\
- IP9(ip, i*32+13, parm); w |= (uint64_t)IPV(ip, i*32+13) << 26;\
- IP9(ip, i*32+14, parm); w |= (uint64_t)IPV(ip, i*32+14) << 28;\
- IP9(ip, i*32+15, parm); w |= (uint64_t)IPV(ip, i*32+15) << 30;\
- IP9(ip, i*32+16, parm); w |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); w |= (uint64_t)IPV(ip, i*32+17) << 34;\
- IP9(ip, i*32+18, parm); w |= (uint64_t)IPV(ip, i*32+18) << 36;\
- IP9(ip, i*32+19, parm); w |= (uint64_t)IPV(ip, i*32+19) << 38;\
- IP9(ip, i*32+20, parm); w |= (uint64_t)IPV(ip, i*32+20) << 40;\
- IP9(ip, i*32+21, parm); w |= (uint64_t)IPV(ip, i*32+21) << 42;\
- IP9(ip, i*32+22, parm); w |= (uint64_t)IPV(ip, i*32+22) << 44;\
- IP9(ip, i*32+23, parm); w |= (uint64_t)IPV(ip, i*32+23) << 46;\
- IP9(ip, i*32+24, parm); w |= (uint64_t)IPV(ip, i*32+24) << 48;\
- IP9(ip, i*32+25, parm); w |= (uint64_t)IPV(ip, i*32+25) << 50;\
- IP9(ip, i*32+26, parm); w |= (uint64_t)IPV(ip, i*32+26) << 52;\
- IP9(ip, i*32+27, parm); w |= (uint64_t)IPV(ip, i*32+27) << 54;\
- IP9(ip, i*32+28, parm); w |= (uint64_t)IPV(ip, i*32+28) << 56;\
- IP9(ip, i*32+29, parm); w |= (uint64_t)IPV(ip, i*32+29) << 58;\
- IP9(ip, i*32+30, parm); w |= (uint64_t)IPV(ip, i*32+30) << 60;\
- IP9(ip, i*32+31, parm); w |= (uint64_t)IPV(ip, i*32+31) << 62;*((uint64_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_2(ip, op, parm) { \
- BITBLK64_2(ip, 0, op, parm); IPI(ip); op += 2*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_3(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 3;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 6;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 9;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 12;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 15;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 18;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 21;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); w |= (uint64_t)IPV(ip, i*64+ 9) << 27;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 30;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 33;\
- IP9(ip, i*64+12, parm); w |= (uint64_t)IPV(ip, i*64+12) << 36;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 39;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 42;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 45;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 48;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 51;\
- IP9(ip, i*64+18, parm); w |= (uint64_t)IPV(ip, i*64+18) << 54;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 57;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 60 | (uint64_t)IPX(ip, i*64+21) << 63;*((uint64_t *)op+i*3+ 0) = w;\
- IP64(ip, i*64+21, parm); w = (uint64_t)IPW(ip, i*64+21) >> 1;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 2;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 5;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); w |= (uint64_t)IPV(ip, i*64+25) << 11;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 14;\
- IP9(ip, i*64+27, parm); w |= (uint64_t)IPV(ip, i*64+27) << 17;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 20;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 23;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 26;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 29;*((uint64_t *)op+i*3+ 1) = w;;\
-}
-
-#define BITPACK64_3(ip, op, parm) { \
- BITBLK64_3(ip, 0, op, parm); IPI(ip); op += 3*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_4(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*16+ 0, parm); w = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); w |= (uint64_t)IPV(ip, i*16+ 1) << 4;\
- IP9(ip, i*16+ 2, parm); w |= (uint64_t)IPV(ip, i*16+ 2) << 8;\
- IP9(ip, i*16+ 3, parm); w |= (uint64_t)IPV(ip, i*16+ 3) << 12;\
- IP9(ip, i*16+ 4, parm); w |= (uint64_t)IPV(ip, i*16+ 4) << 16;\
- IP9(ip, i*16+ 5, parm); w |= (uint64_t)IPV(ip, i*16+ 5) << 20;\
- IP9(ip, i*16+ 6, parm); w |= (uint64_t)IPV(ip, i*16+ 6) << 24;\
- IP9(ip, i*16+ 7, parm); w |= (uint64_t)IPV(ip, i*16+ 7) << 28;\
- IP9(ip, i*16+ 8, parm); w |= (uint64_t)IPV(ip, i*16+ 8) << 32;\
- IP9(ip, i*16+ 9, parm); w |= (uint64_t)IPV(ip, i*16+ 9) << 36;\
- IP9(ip, i*16+10, parm); w |= (uint64_t)IPV(ip, i*16+10) << 40;\
- IP9(ip, i*16+11, parm); w |= (uint64_t)IPV(ip, i*16+11) << 44;\
- IP9(ip, i*16+12, parm); w |= (uint64_t)IPV(ip, i*16+12) << 48;\
- IP9(ip, i*16+13, parm); w |= (uint64_t)IPV(ip, i*16+13) << 52;\
- IP9(ip, i*16+14, parm); w |= (uint64_t)IPV(ip, i*16+14) << 56;\
- IP9(ip, i*16+15, parm); w |= (uint64_t)IPV(ip, i*16+15) << 60;*((uint64_t *)op+i*1+ 0) = w;;\
-}
-
-#define BITPACK64_4(ip, op, parm) { \
- BITBLK64_4(ip, 0, op, parm);\
- BITBLK64_4(ip, 1, op, parm); IPI(ip); op += 4*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_5(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 5;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 10;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 15;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 20;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 25;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 30;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 35;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 40;\
- IP9(ip, i*64+ 9, parm); w |= (uint64_t)IPV(ip, i*64+ 9) << 45;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 50;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 55 | (uint64_t)IPX(ip, i*64+12) << 60;*((uint64_t *)op+i*5+ 0) = w;\
- IP64(ip, i*64+12, parm); w = (uint64_t)IPW(ip, i*64+12) >> 4;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 1;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 6;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 11;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 21;\
- IP9(ip, i*64+18, parm); w |= (uint64_t)IPV(ip, i*64+18) << 26;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 31;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 36;\
- IP9(ip, i*64+21, parm); w |= (uint64_t)IPV(ip, i*64+21) << 41;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 46;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 51;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 56 | (uint64_t)IPX(ip, i*64+25) << 61;*((uint64_t *)op+i*5+ 1) = w;\
- IP64(ip, i*64+25, parm); w = (uint64_t)IPW(ip, i*64+25) >> 3;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 2;\
- IP9(ip, i*64+27, parm); w |= (uint64_t)IPV(ip, i*64+27) << 7;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 12;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 17;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 22;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 27;*((uint64_t *)op+i*5+ 2) = w;;\
-}
-
-#define BITPACK64_5(ip, op, parm) { \
- BITBLK64_5(ip, 0, op, parm); IPI(ip); op += 5*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_6(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*32+ 0, parm); w = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); w |= (uint64_t)IPV(ip, i*32+ 1) << 6;\
- IP9(ip, i*32+ 2, parm); w |= (uint64_t)IPV(ip, i*32+ 2) << 12;\
- IP9(ip, i*32+ 3, parm); w |= (uint64_t)IPV(ip, i*32+ 3) << 18;\
- IP9(ip, i*32+ 4, parm); w |= (uint64_t)IPV(ip, i*32+ 4) << 24;\
- IP9(ip, i*32+ 5, parm); w |= (uint64_t)IPV(ip, i*32+ 5) << 30;\
- IP9(ip, i*32+ 6, parm); w |= (uint64_t)IPV(ip, i*32+ 6) << 36;\
- IP9(ip, i*32+ 7, parm); w |= (uint64_t)IPV(ip, i*32+ 7) << 42;\
- IP9(ip, i*32+ 8, parm); w |= (uint64_t)IPV(ip, i*32+ 8) << 48;\
- IP9(ip, i*32+ 9, parm); w |= (uint64_t)IPV(ip, i*32+ 9) << 54 | (uint64_t)IPX(ip, i*32+10) << 60;*((uint64_t *)op+i*3+ 0) = w;\
- IP64(ip, i*32+10, parm); w = (uint64_t)IPW(ip, i*32+10) >> 4;\
- IP9(ip, i*32+11, parm); w |= (uint64_t)IPV(ip, i*32+11) << 2;\
- IP9(ip, i*32+12, parm); w |= (uint64_t)IPV(ip, i*32+12) << 8;\
- IP9(ip, i*32+13, parm); w |= (uint64_t)IPV(ip, i*32+13) << 14;\
- IP9(ip, i*32+14, parm); w |= (uint64_t)IPV(ip, i*32+14) << 20;\
- IP9(ip, i*32+15, parm); w |= (uint64_t)IPV(ip, i*32+15) << 26;\
- IP9(ip, i*32+16, parm); w |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); w |= (uint64_t)IPV(ip, i*32+17) << 38;\
- IP9(ip, i*32+18, parm); w |= (uint64_t)IPV(ip, i*32+18) << 44;\
- IP9(ip, i*32+19, parm); w |= (uint64_t)IPV(ip, i*32+19) << 50;\
- IP9(ip, i*32+20, parm); w |= (uint64_t)IPV(ip, i*32+20) << 56 | (uint64_t)IPX(ip, i*32+21) << 62;*((uint64_t *)op+i*3+ 1) = w;\
- IP64(ip, i*32+21, parm); w = (uint64_t)IPW(ip, i*32+21) >> 2;\
- IP9(ip, i*32+22, parm); w |= (uint64_t)IPV(ip, i*32+22) << 4;\
- IP9(ip, i*32+23, parm); w |= (uint64_t)IPV(ip, i*32+23) << 10;\
- IP9(ip, i*32+24, parm); w |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); w |= (uint64_t)IPV(ip, i*32+25) << 22;\
- IP9(ip, i*32+26, parm); w |= (uint64_t)IPV(ip, i*32+26) << 28;\
- IP9(ip, i*32+27, parm); w |= (uint64_t)IPV(ip, i*32+27) << 34;\
- IP9(ip, i*32+28, parm); w |= (uint64_t)IPV(ip, i*32+28) << 40;\
- IP9(ip, i*32+29, parm); w |= (uint64_t)IPV(ip, i*32+29) << 46;\
- IP9(ip, i*32+30, parm); w |= (uint64_t)IPV(ip, i*32+30) << 52;\
- IP9(ip, i*32+31, parm); w |= (uint64_t)IPV(ip, i*32+31) << 58;*((uint64_t *)op+i*3+ 2) = w;;\
-}
-
-#define BITPACK64_6(ip, op, parm) { \
- BITBLK64_6(ip, 0, op, parm); IPI(ip); op += 6*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_7(ip, i, op, parm) { uint64_t w;;\
- IP9(ip, i*64+ 0, parm); w = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); w |= (uint64_t)IPV(ip, i*64+ 1) << 7;\
- IP9(ip, i*64+ 2, parm); w |= (uint64_t)IPV(ip, i*64+ 2) << 14;\
- IP9(ip, i*64+ 3, parm); w |= (uint64_t)IPV(ip, i*64+ 3) << 21;\
- IP9(ip, i*64+ 4, parm); w |= (uint64_t)IPV(ip, i*64+ 4) << 28;\
- IP9(ip, i*64+ 5, parm); w |= (uint64_t)IPV(ip, i*64+ 5) << 35;\
- IP9(ip, i*64+ 6, parm); w |= (uint64_t)IPV(ip, i*64+ 6) << 42;\
- IP9(ip, i*64+ 7, parm); w |= (uint64_t)IPV(ip, i*64+ 7) << 49;\
- IP9(ip, i*64+ 8, parm); w |= (uint64_t)IPV(ip, i*64+ 8) << 56 | (uint64_t)IPX(ip, i*64+9) << 63;*((uint64_t *)op+i*7+ 0) = w;\
- IP64(ip, i*64+ 9, parm); w = (uint64_t)IPW(ip, i*64+ 9) >> 1;\
- IP9(ip, i*64+10, parm); w |= (uint64_t)IPV(ip, i*64+10) << 6;\
- IP9(ip, i*64+11, parm); w |= (uint64_t)IPV(ip, i*64+11) << 13;\
- IP9(ip, i*64+12, parm); w |= (uint64_t)IPV(ip, i*64+12) << 20;\
- IP9(ip, i*64+13, parm); w |= (uint64_t)IPV(ip, i*64+13) << 27;\
- IP9(ip, i*64+14, parm); w |= (uint64_t)IPV(ip, i*64+14) << 34;\
- IP9(ip, i*64+15, parm); w |= (uint64_t)IPV(ip, i*64+15) << 41;\
- IP9(ip, i*64+16, parm); w |= (uint64_t)IPV(ip, i*64+16) << 48;\
- IP9(ip, i*64+17, parm); w |= (uint64_t)IPV(ip, i*64+17) << 55 | (uint64_t)IPX(ip, i*64+18) << 62;*((uint64_t *)op+i*7+ 1) = w;\
- IP64(ip, i*64+18, parm); w = (uint64_t)IPW(ip, i*64+18) >> 2;\
- IP9(ip, i*64+19, parm); w |= (uint64_t)IPV(ip, i*64+19) << 5;\
- IP9(ip, i*64+20, parm); w |= (uint64_t)IPV(ip, i*64+20) << 12;\
- IP9(ip, i*64+21, parm); w |= (uint64_t)IPV(ip, i*64+21) << 19;\
- IP9(ip, i*64+22, parm); w |= (uint64_t)IPV(ip, i*64+22) << 26;\
- IP9(ip, i*64+23, parm); w |= (uint64_t)IPV(ip, i*64+23) << 33;\
- IP9(ip, i*64+24, parm); w |= (uint64_t)IPV(ip, i*64+24) << 40;\
- IP9(ip, i*64+25, parm); w |= (uint64_t)IPV(ip, i*64+25) << 47;\
- IP9(ip, i*64+26, parm); w |= (uint64_t)IPV(ip, i*64+26) << 54 | (uint64_t)IPX(ip, i*64+27) << 61;*((uint64_t *)op+i*7+ 2) = w;\
- IP64(ip, i*64+27, parm); w = (uint64_t)IPW(ip, i*64+27) >> 3;\
- IP9(ip, i*64+28, parm); w |= (uint64_t)IPV(ip, i*64+28) << 4;\
- IP9(ip, i*64+29, parm); w |= (uint64_t)IPV(ip, i*64+29) << 11;\
- IP9(ip, i*64+30, parm); w |= (uint64_t)IPV(ip, i*64+30) << 18;\
- IP9(ip, i*64+31, parm); w |= (uint64_t)IPV(ip, i*64+31) << 25;*((uint64_t *)op+i*7+ 3) = w;;\
-}
-
-#define BITPACK64_7(ip, op, parm) { \
- BITBLK64_7(ip, 0, op, parm); IPI(ip); op += 7*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_8(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*1+ 0) = (uint64_t)IPV(ip, i*8+ 0) ;\
- IP9(ip, i*8+ 1, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 1) << 8;\
- IP9(ip, i*8+ 2, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 2) << 16;\
- IP9(ip, i*8+ 3, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 3) << 24;\
- IP9(ip, i*8+ 4, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 4) << 32;\
- IP9(ip, i*8+ 5, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 5) << 40;\
- IP9(ip, i*8+ 6, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 6) << 48;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*1+ 0) |= (uint64_t)IPV(ip, i*8+ 7) << 56;\
-}
-
-#define BITPACK64_8(ip, op, parm) { \
- BITBLK64_8(ip, 0, op, parm);\
- BITBLK64_8(ip, 1, op, parm);\
- BITBLK64_8(ip, 2, op, parm);\
- BITBLK64_8(ip, 3, op, parm); IPI(ip); op += 8*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_9(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 9;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 18;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 27;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 4) << 36;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 5) << 45;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*64+ 6) << 54 | (uint64_t)IPX(ip, i*64+7) << 63;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*64+ 7) >> 1;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+ 9) << 17;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+10) << 26;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+11) << 35;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+12) << 44;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*64+13) << 53 | (uint64_t)IPX(ip, i*64+14) << 62;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*64+14) >> 2;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+15) << 7;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+17) << 25;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+18) << 34;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+19) << 43;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*64+20) << 52 | (uint64_t)IPX(ip, i*64+21) << 61;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*64+21) >> 3;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+22) << 6;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+23) << 15;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+24) << 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+25) << 33;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+26) << 42;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*64+27) << 51 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*64+28) >> 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+29) << 5;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+30) << 14;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*64+31) << 23;\
-}
-
-#define BITPACK64_9(ip, op, parm) { \
- BITBLK64_9(ip, 0, op, parm); IPI(ip); op += 9*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_10(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 10;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 20;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 3) << 30;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 4) << 40;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*32+ 5) << 50 | (uint64_t)IPX(ip, i*32+6) << 60;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*32+ 6) >> 4;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 7) << 6;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+ 9) << 26;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+10) << 36;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*32+11) << 46 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*32+12) >> 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+13) << 2;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+14) << 12;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+15) << 22;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+17) << 42;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*32+18) << 52 | (uint64_t)IPX(ip, i*32+19) << 62;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*32+19) >> 2;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+20) << 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+21) << 18;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+22) << 28;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+23) << 38;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*32+24) << 48 | (uint64_t)IPX(ip, i*32+25) << 58;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*32+25) >> 6;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+26) << 4;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+27) << 14;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+28) << 24;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+29) << 34;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+30) << 44;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*32+31) << 54;\
-}
-
-#define BITPACK64_10(ip, op, parm) { \
- BITBLK64_10(ip, 0, op, parm); IPI(ip); op += 10*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_11(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 11;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 22;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 33;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*64+ 4) << 44 | (uint64_t)IPX(ip, i*64+5) << 55;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*64+ 5) >> 9;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 2;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 13;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+ 9) << 35;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*64+10) << 46 | (uint64_t)IPX(ip, i*64+11) << 57;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*64+11) >> 7;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+12) << 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+13) << 15;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+14) << 26;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+15) << 37;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*64+16) << 48 | (uint64_t)IPX(ip, i*64+17) << 59;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*64+17) >> 5;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+18) << 6;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+19) << 17;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+20) << 28;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+21) << 39;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*64+22) << 50 | (uint64_t)IPX(ip, i*64+23) << 61;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*64+23) >> 3;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+25) << 19;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+26) << 30;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+27) << 41;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*64+28) << 52 | (uint64_t)IPX(ip, i*64+29) << 63;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*64+29) >> 1;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*64+30) << 10;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*64+31) << 21;\
-}
-
-#define BITPACK64_11(ip, op, parm) { \
- BITBLK64_11(ip, 0, op, parm); IPI(ip); op += 11*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_12(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 12;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 2) << 24;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 3) << 36;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*16+ 4) << 48 | (uint64_t)IPX(ip, i*16+5) << 60;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*16+ 5) >> 4;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 6) << 8;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 7) << 20;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 8) << 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*16+ 9) << 44 | (uint64_t)IPX(ip, i*16+10) << 56;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*16+10) >> 8;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+11) << 4;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+12) << 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+13) << 28;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+14) << 40;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*16+15) << 52;\
-}
-
-#define BITPACK64_12(ip, op, parm) { \
- BITBLK64_12(ip, 0, op, parm);\
- BITBLK64_12(ip, 1, op, parm); IPI(ip); op += 12*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_13(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 13;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 26;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 39 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*64+ 4) >> 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 1;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 14;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 27;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*64+ 8) << 40 | (uint64_t)IPX(ip, i*64+9) << 53;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*64+ 9) >> 11;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+10) << 2;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+11) << 15;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+12) << 28;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*64+13) << 41 | (uint64_t)IPX(ip, i*64+14) << 54;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*64+14) >> 10;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+15) << 3;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+17) << 29;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*64+18) << 42 | (uint64_t)IPX(ip, i*64+19) << 55;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*64+19) >> 9;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+20) << 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+21) << 17;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+22) << 30;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*64+23) << 43 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+25) << 5;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+26) << 18;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+27) << 31;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*64+28) << 44 | (uint64_t)IPX(ip, i*64+29) << 57;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*64+29) >> 7;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*64+30) << 6;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*64+31) << 19;\
-}
-
-#define BITPACK64_13(ip, op, parm) { \
- BITBLK64_13(ip, 0, op, parm); IPI(ip); op += 13*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_14(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 14;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 28;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*32+ 3) << 42 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*32+ 4) >> 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 5) << 6;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 6) << 20;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 7) << 34;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*32+ 8) << 48 | (uint64_t)IPX(ip, i*32+9) << 62;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*32+ 9) >> 2;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+10) << 12;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+11) << 26;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*32+12) << 40 | (uint64_t)IPX(ip, i*32+13) << 54;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*32+13) >> 10;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+14) << 4;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+15) << 18;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+16) << 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*32+17) << 46 | (uint64_t)IPX(ip, i*32+18) << 60;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*32+18) >> 4;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+19) << 10;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+20) << 24;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*32+21) << 38 | (uint64_t)IPX(ip, i*32+22) << 52;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*32+22) >> 12;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+23) << 2;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+25) << 30;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*32+26) << 44 | (uint64_t)IPX(ip, i*32+27) << 58;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*32+27) >> 6;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+28) << 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+29) << 22;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+30) << 36;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*32+31) << 50;\
-}
-
-#define BITPACK64_14(ip, op, parm) { \
- BITBLK64_14(ip, 0, op, parm); IPI(ip); op += 14*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_15(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 15;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 30;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*64+ 3) << 45 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*64+ 4) >> 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 11;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 26;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*64+ 7) << 41 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 7;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+10) << 22;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*64+11) << 37 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*64+12) >> 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+13) << 3;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+14) << 18;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+15) << 33;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*64+16) << 48 | (uint64_t)IPX(ip, i*64+17) << 63;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*64+17) >> 1;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+18) << 14;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+19) << 29;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*64+20) << 44 | (uint64_t)IPX(ip, i*64+21) << 59;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*64+21) >> 5;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+22) << 10;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+23) << 25;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*64+24) << 40 | (uint64_t)IPX(ip, i*64+25) << 55;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*64+25) >> 9;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+26) << 6;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+27) << 21;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*64+28) << 36 | (uint64_t)IPX(ip, i*64+29) << 51;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*64+29) >> 13;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*64+30) << 2;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*64+31) << 17;\
-}
-
-#define BITPACK64_15(ip, op, parm) { \
- BITBLK64_15(ip, 0, op, parm); IPI(ip); op += 15*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_16(ip, i, op, parm) { \
- IP9(ip, i*4+ 0, parm); *(uint16_t *)(op+i*8+ 0) = IPV(ip, i*4+ 0);\
- IP9(ip, i*4+ 1, parm); *(uint16_t *)(op+i*8+ 2) = IPV(ip, i*4+ 1);\
- IP9(ip, i*4+ 2, parm); *(uint16_t *)(op+i*8+ 4) = IPV(ip, i*4+ 2);\
- IP9(ip, i*4+ 3, parm); *(uint16_t *)(op+i*8+ 6) = IPV(ip, i*4+ 3);;\
-}
-
-#define BITPACK64_16(ip, op, parm) { \
- BITBLK64_16(ip, 0, op, parm);\
- BITBLK64_16(ip, 1, op, parm);\
- BITBLK64_16(ip, 2, op, parm);\
- BITBLK64_16(ip, 3, op, parm);\
- BITBLK64_16(ip, 4, op, parm);\
- BITBLK64_16(ip, 5, op, parm);\
- BITBLK64_16(ip, 6, op, parm);\
- BITBLK64_16(ip, 7, op, parm); IPI(ip); op += 16*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_17(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*17+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*17+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 17;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*17+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 34 | (uint64_t)IPX(ip, i*64+3) << 51;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*17+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 13;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 21;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*64+ 6) << 38 | (uint64_t)IPX(ip, i*64+7) << 55;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*17+ 2) = (uint64_t)IPW(ip, i*64+ 7) >> 9;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 25;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*64+10) << 42 | (uint64_t)IPX(ip, i*64+11) << 59;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*17+ 3) = (uint64_t)IPW(ip, i*64+11) >> 5;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+12) << 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+13) << 29;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*64+14) << 46 | (uint64_t)IPX(ip, i*64+15) << 63;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*17+ 4) = (uint64_t)IPW(ip, i*64+15) >> 1;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*64+17) << 33 | (uint64_t)IPX(ip, i*64+18) << 50;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*17+ 5) = (uint64_t)IPW(ip, i*64+18) >> 14;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+19) << 3;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+20) << 20;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*64+21) << 37 | (uint64_t)IPX(ip, i*64+22) << 54;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*17+ 6) = (uint64_t)IPW(ip, i*64+22) >> 10;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+23) << 7;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+24) << 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*64+25) << 41 | (uint64_t)IPX(ip, i*64+26) << 58;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*17+ 7) = (uint64_t)IPW(ip, i*64+26) >> 6;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+27) << 11;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+28) << 28;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*64+29) << 45 | (uint64_t)IPX(ip, i*64+30) << 62;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*17+ 8) = (uint64_t)IPW(ip, i*64+30) >> 2;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*17+ 8) |= (uint64_t)IPV(ip, i*64+31) << 15;\
-}
-
-#define BITPACK64_17(ip, op, parm) { \
- BITBLK64_17(ip, 0, op, parm); IPI(ip); op += 17*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_18(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 18;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*9+ 0) |= (uint64_t)IPV(ip, i*32+ 2) << 36 | (uint64_t)IPX(ip, i*32+3) << 54;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*32+ 3) >> 10;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 4) << 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 5) << 26;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*32+ 6) << 44 | (uint64_t)IPX(ip, i*32+7) << 62;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*32+ 7) >> 2;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*32+ 8) << 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*32+ 9) << 34 | (uint64_t)IPX(ip, i*32+10) << 52;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*32+10) >> 12;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+11) << 6;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+12) << 24;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*32+13) << 42 | (uint64_t)IPX(ip, i*32+14) << 60;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*32+14) >> 4;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*32+15) << 14;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*9+ 4) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 50;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*9+ 5) = (uint64_t)IPW(ip, i*32+17) >> 14;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+18) << 4;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+19) << 22;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*32+20) << 40 | (uint64_t)IPX(ip, i*32+21) << 58;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*9+ 6) = (uint64_t)IPW(ip, i*32+21) >> 6;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*32+22) << 12;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*32+23) << 30 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*9+ 7) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+25) << 2;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+26) << 20;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*32+27) << 38 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*9+ 8) = (uint64_t)IPW(ip, i*32+28) >> 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+29) << 10;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+30) << 28;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*32+31) << 46;\
-}
-
-#define BITPACK64_18(ip, op, parm) { \
- BITBLK64_18(ip, 0, op, parm); IPI(ip); op += 18*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_19(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*19+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*19+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 19;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*19+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 38 | (uint64_t)IPX(ip, i*64+3) << 57;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*19+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 7;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 31 | (uint64_t)IPX(ip, i*64+6) << 50;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*19+ 2) = (uint64_t)IPW(ip, i*64+ 6) >> 14;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 5;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*64+ 9) << 43 | (uint64_t)IPX(ip, i*64+10) << 62;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*19+ 3) = (uint64_t)IPW(ip, i*64+10) >> 2;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*19+ 3) |= (uint64_t)IPV(ip, i*64+11) << 17;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*19+ 3) |= (uint64_t)IPV(ip, i*64+12) << 36 | (uint64_t)IPX(ip, i*64+13) << 55;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*19+ 4) = (uint64_t)IPW(ip, i*64+13) >> 9;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*64+14) << 10;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*64+15) << 29 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*19+ 5) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+17) << 3;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+18) << 22;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*64+19) << 41 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*19+ 6) = (uint64_t)IPW(ip, i*64+20) >> 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*19+ 6) |= (uint64_t)IPV(ip, i*64+21) << 15;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*19+ 6) |= (uint64_t)IPV(ip, i*64+22) << 34 | (uint64_t)IPX(ip, i*64+23) << 53;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*19+ 7) = (uint64_t)IPW(ip, i*64+23) >> 11;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*64+25) << 27 | (uint64_t)IPX(ip, i*64+26) << 46;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*19+ 8) = (uint64_t)IPW(ip, i*64+26) >> 18;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+27) << 1;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+28) << 20;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*64+29) << 39 | (uint64_t)IPX(ip, i*64+30) << 58;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*19+ 9) = (uint64_t)IPW(ip, i*64+30) >> 6;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*19+ 9) |= (uint64_t)IPV(ip, i*64+31) << 13;\
-}
-
-#define BITPACK64_19(ip, op, parm) { \
- BITBLK64_19(ip, 0, op, parm); IPI(ip); op += 19*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_20(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 20;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*5+ 0) |= (uint64_t)IPV(ip, i*16+ 2) << 40 | (uint64_t)IPX(ip, i*16+3) << 60;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*16+ 3) >> 4;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*16+ 4) << 16;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*16+ 5) << 36 | (uint64_t)IPX(ip, i*16+6) << 56;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*16+ 6) >> 8;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*16+ 7) << 12;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*5+ 2) |= (uint64_t)IPV(ip, i*16+ 8) << 32 | (uint64_t)IPX(ip, i*16+9) << 52;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*16+ 9) >> 12;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*16+10) << 8;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*16+11) << 28 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*16+12) >> 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+13) << 4;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+14) << 24;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*16+15) << 44;\
-}
-
-#define BITPACK64_20(ip, op, parm) { \
- BITBLK64_20(ip, 0, op, parm);\
- BITBLK64_20(ip, 1, op, parm); IPI(ip); op += 20*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_21(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*21+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*21+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 21;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*21+ 0) |= (uint64_t)IPV(ip, i*64+ 2) << 42 | (uint64_t)IPX(ip, i*64+3) << 63;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*21+ 1) = (uint64_t)IPW(ip, i*64+ 3) >> 1;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 20;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*64+ 5) << 41 | (uint64_t)IPX(ip, i*64+6) << 62;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*21+ 2) = (uint64_t)IPW(ip, i*64+ 6) >> 2;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*21+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 19;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*21+ 2) |= (uint64_t)IPV(ip, i*64+ 8) << 40 | (uint64_t)IPX(ip, i*64+9) << 61;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*21+ 3) = (uint64_t)IPW(ip, i*64+ 9) >> 3;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*64+10) << 18;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*64+11) << 39 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*21+ 4) = (uint64_t)IPW(ip, i*64+12) >> 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*21+ 4) |= (uint64_t)IPV(ip, i*64+13) << 17;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*21+ 4) |= (uint64_t)IPV(ip, i*64+14) << 38 | (uint64_t)IPX(ip, i*64+15) << 59;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*21+ 5) = (uint64_t)IPW(ip, i*64+15) >> 5;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*64+16) << 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*64+17) << 37 | (uint64_t)IPX(ip, i*64+18) << 58;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*21+ 6) = (uint64_t)IPW(ip, i*64+18) >> 6;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*21+ 6) |= (uint64_t)IPV(ip, i*64+19) << 15;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*21+ 6) |= (uint64_t)IPV(ip, i*64+20) << 36 | (uint64_t)IPX(ip, i*64+21) << 57;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*21+ 7) = (uint64_t)IPW(ip, i*64+21) >> 7;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*64+22) << 14;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*64+23) << 35 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*21+ 8) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*21+ 8) |= (uint64_t)IPV(ip, i*64+25) << 13;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*21+ 8) |= (uint64_t)IPV(ip, i*64+26) << 34 | (uint64_t)IPX(ip, i*64+27) << 55;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*21+ 9) = (uint64_t)IPW(ip, i*64+27) >> 9;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*64+28) << 12;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*64+29) << 33 | (uint64_t)IPX(ip, i*64+30) << 54;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*21+10) = (uint64_t)IPW(ip, i*64+30) >> 10;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*21+10) |= (uint64_t)IPV(ip, i*64+31) << 11;\
-}
-
-#define BITPACK64_21(ip, op, parm) { \
- BITBLK64_21(ip, 0, op, parm); IPI(ip); op += 21*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_22(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*11+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 22 | (uint64_t)IPX(ip, i*32+2) << 44;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 20;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 2;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*11+ 1) |= (uint64_t)IPV(ip, i*32+ 4) << 24 | (uint64_t)IPX(ip, i*32+5) << 46;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*32+ 5) >> 18;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*32+ 6) << 4;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*32+ 7) << 26 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*32+ 9) << 6;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*11+ 3) |= (uint64_t)IPV(ip, i*32+10) << 28 | (uint64_t)IPX(ip, i*32+11) << 50;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*32+11) >> 14;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*32+12) << 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*32+13) << 30 | (uint64_t)IPX(ip, i*32+14) << 52;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*32+14) >> 12;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*32+15) << 10;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*11+ 5) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 54;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*11+ 6) = (uint64_t)IPW(ip, i*32+17) >> 10;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*32+18) << 12;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*32+19) << 34 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*11+ 7) = (uint64_t)IPW(ip, i*32+20) >> 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*11+ 7) |= (uint64_t)IPV(ip, i*32+21) << 14;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*11+ 7) |= (uint64_t)IPV(ip, i*32+22) << 36 | (uint64_t)IPX(ip, i*32+23) << 58;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*11+ 8) = (uint64_t)IPW(ip, i*32+23) >> 6;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*32+24) << 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*32+25) << 38 | (uint64_t)IPX(ip, i*32+26) << 60;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*11+ 9) = (uint64_t)IPW(ip, i*32+26) >> 4;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*11+ 9) |= (uint64_t)IPV(ip, i*32+27) << 18;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*11+ 9) |= (uint64_t)IPV(ip, i*32+28) << 40 | (uint64_t)IPX(ip, i*32+29) << 62;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*11+10) = (uint64_t)IPW(ip, i*32+29) >> 2;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*32+30) << 20;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*32+31) << 42;\
-}
-
-#define BITPACK64_22(ip, op, parm) { \
- BITBLK64_22(ip, 0, op, parm); IPI(ip); op += 22*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_23(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*23+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*23+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 23 | (uint64_t)IPX(ip, i*64+2) << 46;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*23+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 18;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*23+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 5;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*23+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 28 | (uint64_t)IPX(ip, i*64+5) << 51;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*23+ 2) = (uint64_t)IPW(ip, i*64+ 5) >> 13;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 10;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*64+ 7) << 33 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*23+ 3) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*23+ 3) |= (uint64_t)IPV(ip, i*64+ 9) << 15;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*23+ 3) |= (uint64_t)IPV(ip, i*64+10) << 38 | (uint64_t)IPX(ip, i*64+11) << 61;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*23+ 4) = (uint64_t)IPW(ip, i*64+11) >> 3;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*23+ 4) |= (uint64_t)IPV(ip, i*64+12) << 20 | (uint64_t)IPX(ip, i*64+13) << 43;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*23+ 5) = (uint64_t)IPW(ip, i*64+13) >> 21;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*64+14) << 2;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*64+15) << 25 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*23+ 6) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*23+ 6) |= (uint64_t)IPV(ip, i*64+17) << 7;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*23+ 6) |= (uint64_t)IPV(ip, i*64+18) << 30 | (uint64_t)IPX(ip, i*64+19) << 53;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*23+ 7) = (uint64_t)IPW(ip, i*64+19) >> 11;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*64+20) << 12;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*64+21) << 35 | (uint64_t)IPX(ip, i*64+22) << 58;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*23+ 8) = (uint64_t)IPW(ip, i*64+22) >> 6;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*23+ 8) |= (uint64_t)IPV(ip, i*64+23) << 17;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*23+ 8) |= (uint64_t)IPV(ip, i*64+24) << 40 | (uint64_t)IPX(ip, i*64+25) << 63;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*23+ 9) = (uint64_t)IPW(ip, i*64+25) >> 1;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*23+ 9) |= (uint64_t)IPV(ip, i*64+26) << 22 | (uint64_t)IPX(ip, i*64+27) << 45;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*23+10) = (uint64_t)IPW(ip, i*64+27) >> 19;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*64+28) << 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*64+29) << 27 | (uint64_t)IPX(ip, i*64+30) << 50;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*23+11) = (uint64_t)IPW(ip, i*64+30) >> 14;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*23+11) |= (uint64_t)IPV(ip, i*64+31) << 9;\
-}
-
-#define BITPACK64_23(ip, op, parm) { \
- BITBLK64_23(ip, 0, op, parm); IPI(ip); op += 23*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_24(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*8+ 0) ;\
- IP9(ip, i*8+ 1, parm); *((uint64_t *)op+i*3+ 0) |= (uint64_t)IPV(ip, i*8+ 1) << 24 | (uint64_t)IPX(ip, i*8+2) << 48;\
- IP64(ip, i*8+ 2, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*8+ 2) >> 16;\
- IP9(ip, i*8+ 3, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*8+ 3) << 8;\
- IP9(ip, i*8+ 4, parm); *((uint64_t *)op+i*3+ 1) |= (uint64_t)IPV(ip, i*8+ 4) << 32 | (uint64_t)IPX(ip, i*8+5) << 56;\
- IP64(ip, i*8+ 5, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*8+ 5) >> 8;\
- IP9(ip, i*8+ 6, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*8+ 6) << 16;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*8+ 7) << 40;\
-}
-
-#define BITPACK64_24(ip, op, parm) { \
- BITBLK64_24(ip, 0, op, parm);\
- BITBLK64_24(ip, 1, op, parm);\
- BITBLK64_24(ip, 2, op, parm);\
- BITBLK64_24(ip, 3, op, parm); IPI(ip); op += 24*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_25(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*25+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*25+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 25 | (uint64_t)IPX(ip, i*64+2) << 50;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*25+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 14;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*25+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 11;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*25+ 1) |= (uint64_t)IPV(ip, i*64+ 4) << 36 | (uint64_t)IPX(ip, i*64+5) << 61;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*25+ 2) = (uint64_t)IPW(ip, i*64+ 5) >> 3;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*25+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 22 | (uint64_t)IPX(ip, i*64+7) << 47;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*25+ 3) = (uint64_t)IPW(ip, i*64+ 7) >> 17;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*64+ 8) << 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*64+ 9) << 33 | (uint64_t)IPX(ip, i*64+10) << 58;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*25+ 4) = (uint64_t)IPW(ip, i*64+10) >> 6;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*25+ 4) |= (uint64_t)IPV(ip, i*64+11) << 19 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*25+ 5) = (uint64_t)IPW(ip, i*64+12) >> 20;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*25+ 5) |= (uint64_t)IPV(ip, i*64+13) << 5;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*25+ 5) |= (uint64_t)IPV(ip, i*64+14) << 30 | (uint64_t)IPX(ip, i*64+15) << 55;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*25+ 6) = (uint64_t)IPW(ip, i*64+15) >> 9;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*25+ 6) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 41;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*25+ 7) = (uint64_t)IPW(ip, i*64+17) >> 23;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*64+18) << 2;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*64+19) << 27 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*25+ 8) = (uint64_t)IPW(ip, i*64+20) >> 12;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*25+ 8) |= (uint64_t)IPV(ip, i*64+21) << 13;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*25+ 8) |= (uint64_t)IPV(ip, i*64+22) << 38 | (uint64_t)IPX(ip, i*64+23) << 63;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*25+ 9) = (uint64_t)IPW(ip, i*64+23) >> 1;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*25+ 9) |= (uint64_t)IPV(ip, i*64+24) << 24 | (uint64_t)IPX(ip, i*64+25) << 49;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*25+10) = (uint64_t)IPW(ip, i*64+25) >> 15;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*64+26) << 10;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*64+27) << 35 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*25+11) = (uint64_t)IPW(ip, i*64+28) >> 4;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*25+11) |= (uint64_t)IPV(ip, i*64+29) << 21 | (uint64_t)IPX(ip, i*64+30) << 46;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*25+12) = (uint64_t)IPW(ip, i*64+30) >> 18;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*25+12) |= (uint64_t)IPV(ip, i*64+31) << 7;\
-}
-
-#define BITPACK64_25(ip, op, parm) { \
- BITBLK64_25(ip, 0, op, parm); IPI(ip); op += 25*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_26(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*13+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 26 | (uint64_t)IPX(ip, i*32+2) << 52;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 12;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*13+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 14 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*32+ 4) >> 24;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*32+ 5) << 2;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*13+ 2) |= (uint64_t)IPV(ip, i*32+ 6) << 28 | (uint64_t)IPX(ip, i*32+7) << 54;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*32+ 7) >> 10;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*13+ 3) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 42;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*32+ 9) >> 22;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*32+10) << 4;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*32+11) << 30 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*32+12) >> 8;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*13+ 5) |= (uint64_t)IPV(ip, i*32+13) << 18 | (uint64_t)IPX(ip, i*32+14) << 44;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*32+14) >> 20;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*32+15) << 6;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*13+ 6) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 58;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*13+ 7) = (uint64_t)IPW(ip, i*32+17) >> 6;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*13+ 7) |= (uint64_t)IPV(ip, i*32+18) << 20 | (uint64_t)IPX(ip, i*32+19) << 46;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*13+ 8) = (uint64_t)IPW(ip, i*32+19) >> 18;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*32+20) << 8;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*32+21) << 34 | (uint64_t)IPX(ip, i*32+22) << 60;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*13+ 9) = (uint64_t)IPW(ip, i*32+22) >> 4;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*13+ 9) |= (uint64_t)IPV(ip, i*32+23) << 22 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*13+10) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*13+10) |= (uint64_t)IPV(ip, i*32+25) << 10;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*13+10) |= (uint64_t)IPV(ip, i*32+26) << 36 | (uint64_t)IPX(ip, i*32+27) << 62;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*13+11) = (uint64_t)IPW(ip, i*32+27) >> 2;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*13+11) |= (uint64_t)IPV(ip, i*32+28) << 24 | (uint64_t)IPX(ip, i*32+29) << 50;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*13+12) = (uint64_t)IPW(ip, i*32+29) >> 14;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*32+30) << 12;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*32+31) << 38;\
-}
-
-#define BITPACK64_26(ip, op, parm) { \
- BITBLK64_26(ip, 0, op, parm); IPI(ip); op += 26*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_27(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*27+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*27+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 27 | (uint64_t)IPX(ip, i*64+2) << 54;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*27+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 10;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*27+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 17 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*27+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 20;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*27+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 7;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*27+ 2) |= (uint64_t)IPV(ip, i*64+ 6) << 34 | (uint64_t)IPX(ip, i*64+7) << 61;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*27+ 3) = (uint64_t)IPW(ip, i*64+ 7) >> 3;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*27+ 3) |= (uint64_t)IPV(ip, i*64+ 8) << 24 | (uint64_t)IPX(ip, i*64+9) << 51;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*27+ 4) = (uint64_t)IPW(ip, i*64+ 9) >> 13;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*27+ 4) |= (uint64_t)IPV(ip, i*64+10) << 14 | (uint64_t)IPX(ip, i*64+11) << 41;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*27+ 5) = (uint64_t)IPW(ip, i*64+11) >> 23;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*64+12) << 4;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*64+13) << 31 | (uint64_t)IPX(ip, i*64+14) << 58;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*27+ 6) = (uint64_t)IPW(ip, i*64+14) >> 6;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*27+ 6) |= (uint64_t)IPV(ip, i*64+15) << 21 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*27+ 7) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*27+ 7) |= (uint64_t)IPV(ip, i*64+17) << 11 | (uint64_t)IPX(ip, i*64+18) << 38;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*27+ 8) = (uint64_t)IPW(ip, i*64+18) >> 26;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*27+ 8) |= (uint64_t)IPV(ip, i*64+19) << 1;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*27+ 8) |= (uint64_t)IPV(ip, i*64+20) << 28 | (uint64_t)IPX(ip, i*64+21) << 55;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*27+ 9) = (uint64_t)IPW(ip, i*64+21) >> 9;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*27+ 9) |= (uint64_t)IPV(ip, i*64+22) << 18 | (uint64_t)IPX(ip, i*64+23) << 45;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*27+10) = (uint64_t)IPW(ip, i*64+23) >> 19;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*64+24) << 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*64+25) << 35 | (uint64_t)IPX(ip, i*64+26) << 62;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*27+11) = (uint64_t)IPW(ip, i*64+26) >> 2;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*27+11) |= (uint64_t)IPV(ip, i*64+27) << 25 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*27+12) = (uint64_t)IPW(ip, i*64+28) >> 12;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*27+12) |= (uint64_t)IPV(ip, i*64+29) << 15 | (uint64_t)IPX(ip, i*64+30) << 42;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*27+13) = (uint64_t)IPW(ip, i*64+30) >> 22;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*27+13) |= (uint64_t)IPV(ip, i*64+31) << 5;\
-}
-
-#define BITPACK64_27(ip, op, parm) { \
- BITBLK64_27(ip, 0, op, parm); IPI(ip); op += 27*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_28(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*16+ 0) ;\
- IP9(ip, i*16+ 1, parm); *((uint64_t *)op+i*7+ 0) |= (uint64_t)IPV(ip, i*16+ 1) << 28 | (uint64_t)IPX(ip, i*16+2) << 56;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*16+ 2) >> 8;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*7+ 1) |= (uint64_t)IPV(ip, i*16+ 3) << 20 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*16+ 4) >> 16;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*7+ 2) |= (uint64_t)IPV(ip, i*16+ 5) << 12 | (uint64_t)IPX(ip, i*16+6) << 40;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*16+ 6) >> 24;\
- IP9(ip, i*16+ 7, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*16+ 7) << 4;\
- IP9(ip, i*16+ 8, parm); *((uint64_t *)op+i*7+ 3) |= (uint64_t)IPV(ip, i*16+ 8) << 32 | (uint64_t)IPX(ip, i*16+9) << 60;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*16+ 9) >> 4;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*7+ 4) |= (uint64_t)IPV(ip, i*16+10) << 24 | (uint64_t)IPX(ip, i*16+11) << 52;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*16+11) >> 12;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*7+ 5) |= (uint64_t)IPV(ip, i*16+12) << 16 | (uint64_t)IPX(ip, i*16+13) << 44;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*16+13) >> 20;\
- IP9(ip, i*16+14, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*16+14) << 8;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*16+15) << 36;\
-}
-
-#define BITPACK64_28(ip, op, parm) { \
- BITBLK64_28(ip, 0, op, parm);\
- BITBLK64_28(ip, 1, op, parm); IPI(ip); op += 28*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_29(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*29+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*29+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 29 | (uint64_t)IPX(ip, i*64+2) << 58;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*29+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 6;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*29+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 23 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*29+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 12;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*29+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 17 | (uint64_t)IPX(ip, i*64+6) << 46;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*29+ 3) = (uint64_t)IPW(ip, i*64+ 6) >> 18;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*29+ 3) |= (uint64_t)IPV(ip, i*64+ 7) << 11 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*29+ 4) = (uint64_t)IPW(ip, i*64+ 8) >> 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*29+ 4) |= (uint64_t)IPV(ip, i*64+ 9) << 5;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*29+ 4) |= (uint64_t)IPV(ip, i*64+10) << 34 | (uint64_t)IPX(ip, i*64+11) << 63;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*29+ 5) = (uint64_t)IPW(ip, i*64+11) >> 1;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*29+ 5) |= (uint64_t)IPV(ip, i*64+12) << 28 | (uint64_t)IPX(ip, i*64+13) << 57;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*29+ 6) = (uint64_t)IPW(ip, i*64+13) >> 7;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*29+ 6) |= (uint64_t)IPV(ip, i*64+14) << 22 | (uint64_t)IPX(ip, i*64+15) << 51;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*29+ 7) = (uint64_t)IPW(ip, i*64+15) >> 13;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*29+ 7) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 45;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*29+ 8) = (uint64_t)IPW(ip, i*64+17) >> 19;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*29+ 8) |= (uint64_t)IPV(ip, i*64+18) << 10 | (uint64_t)IPX(ip, i*64+19) << 39;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*29+ 9) = (uint64_t)IPW(ip, i*64+19) >> 25;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*64+20) << 4;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*64+21) << 33 | (uint64_t)IPX(ip, i*64+22) << 62;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*29+10) = (uint64_t)IPW(ip, i*64+22) >> 2;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*29+10) |= (uint64_t)IPV(ip, i*64+23) << 27 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*29+11) = (uint64_t)IPW(ip, i*64+24) >> 8;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*29+11) |= (uint64_t)IPV(ip, i*64+25) << 21 | (uint64_t)IPX(ip, i*64+26) << 50;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*29+12) = (uint64_t)IPW(ip, i*64+26) >> 14;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*29+12) |= (uint64_t)IPV(ip, i*64+27) << 15 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*29+13) = (uint64_t)IPW(ip, i*64+28) >> 20;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*29+13) |= (uint64_t)IPV(ip, i*64+29) << 9 | (uint64_t)IPX(ip, i*64+30) << 38;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*29+14) = (uint64_t)IPW(ip, i*64+30) >> 26;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*29+14) |= (uint64_t)IPV(ip, i*64+31) << 3;\
-}
-
-#define BITPACK64_29(ip, op, parm) { \
- BITBLK64_29(ip, 0, op, parm); IPI(ip); op += 29*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_30(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*32+ 0) ;\
- IP9(ip, i*32+ 1, parm); *((uint64_t *)op+i*15+ 0) |= (uint64_t)IPV(ip, i*32+ 1) << 30 | (uint64_t)IPX(ip, i*32+2) << 60;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*32+ 2) >> 4;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*15+ 1) |= (uint64_t)IPV(ip, i*32+ 3) << 26 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*32+ 4) >> 8;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*15+ 2) |= (uint64_t)IPV(ip, i*32+ 5) << 22 | (uint64_t)IPX(ip, i*32+6) << 52;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*32+ 6) >> 12;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*15+ 3) |= (uint64_t)IPV(ip, i*32+ 7) << 18 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*15+ 4) |= (uint64_t)IPV(ip, i*32+ 9) << 14 | (uint64_t)IPX(ip, i*32+10) << 44;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*32+10) >> 20;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*15+ 5) |= (uint64_t)IPV(ip, i*32+11) << 10 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*32+12) >> 24;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*15+ 6) |= (uint64_t)IPV(ip, i*32+13) << 6 | (uint64_t)IPX(ip, i*32+14) << 36;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*32+14) >> 28;\
- IP9(ip, i*32+15, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*32+15) << 2;\
- IP9(ip, i*32+16, parm); *((uint64_t *)op+i*15+ 7) |= (uint64_t)IPV(ip, i*32+16) << 32 | (uint64_t)IPX(ip, i*32+17) << 62;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*15+ 8) = (uint64_t)IPW(ip, i*32+17) >> 2;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*15+ 8) |= (uint64_t)IPV(ip, i*32+18) << 28 | (uint64_t)IPX(ip, i*32+19) << 58;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*15+ 9) = (uint64_t)IPW(ip, i*32+19) >> 6;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*15+ 9) |= (uint64_t)IPV(ip, i*32+20) << 24 | (uint64_t)IPX(ip, i*32+21) << 54;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*15+10) = (uint64_t)IPW(ip, i*32+21) >> 10;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*15+10) |= (uint64_t)IPV(ip, i*32+22) << 20 | (uint64_t)IPX(ip, i*32+23) << 50;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*15+11) = (uint64_t)IPW(ip, i*32+23) >> 14;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*15+11) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 46;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*15+12) = (uint64_t)IPW(ip, i*32+25) >> 18;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*15+12) |= (uint64_t)IPV(ip, i*32+26) << 12 | (uint64_t)IPX(ip, i*32+27) << 42;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*15+13) = (uint64_t)IPW(ip, i*32+27) >> 22;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*15+13) |= (uint64_t)IPV(ip, i*32+28) << 8 | (uint64_t)IPX(ip, i*32+29) << 38;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*15+14) = (uint64_t)IPW(ip, i*32+29) >> 26;\
- IP9(ip, i*32+30, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*32+30) << 4;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*32+31) << 34;\
-}
-
-#define BITPACK64_30(ip, op, parm) { \
- BITBLK64_30(ip, 0, op, parm); IPI(ip); op += 30*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_31(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*31+ 0) = (uint64_t)IPV(ip, i*64+ 0) ;\
- IP9(ip, i*64+ 1, parm); *((uint64_t *)op+i*31+ 0) |= (uint64_t)IPV(ip, i*64+ 1) << 31 | (uint64_t)IPX(ip, i*64+2) << 62;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*31+ 1) = (uint64_t)IPW(ip, i*64+ 2) >> 2;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*31+ 1) |= (uint64_t)IPV(ip, i*64+ 3) << 29 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*31+ 2) = (uint64_t)IPW(ip, i*64+ 4) >> 4;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*31+ 2) |= (uint64_t)IPV(ip, i*64+ 5) << 27 | (uint64_t)IPX(ip, i*64+6) << 58;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*31+ 3) = (uint64_t)IPW(ip, i*64+ 6) >> 6;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*31+ 3) |= (uint64_t)IPV(ip, i*64+ 7) << 25 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*31+ 4) = (uint64_t)IPW(ip, i*64+ 8) >> 8;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*31+ 4) |= (uint64_t)IPV(ip, i*64+ 9) << 23 | (uint64_t)IPX(ip, i*64+10) << 54;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*31+ 5) = (uint64_t)IPW(ip, i*64+10) >> 10;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*31+ 5) |= (uint64_t)IPV(ip, i*64+11) << 21 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*31+ 6) = (uint64_t)IPW(ip, i*64+12) >> 12;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*31+ 6) |= (uint64_t)IPV(ip, i*64+13) << 19 | (uint64_t)IPX(ip, i*64+14) << 50;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*31+ 7) = (uint64_t)IPW(ip, i*64+14) >> 14;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*31+ 7) |= (uint64_t)IPV(ip, i*64+15) << 17 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*31+ 8) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*31+ 8) |= (uint64_t)IPV(ip, i*64+17) << 15 | (uint64_t)IPX(ip, i*64+18) << 46;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*31+ 9) = (uint64_t)IPW(ip, i*64+18) >> 18;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*31+ 9) |= (uint64_t)IPV(ip, i*64+19) << 13 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*31+10) = (uint64_t)IPW(ip, i*64+20) >> 20;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*31+10) |= (uint64_t)IPV(ip, i*64+21) << 11 | (uint64_t)IPX(ip, i*64+22) << 42;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*31+11) = (uint64_t)IPW(ip, i*64+22) >> 22;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*31+11) |= (uint64_t)IPV(ip, i*64+23) << 9 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*31+12) = (uint64_t)IPW(ip, i*64+24) >> 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*31+12) |= (uint64_t)IPV(ip, i*64+25) << 7 | (uint64_t)IPX(ip, i*64+26) << 38;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*31+13) = (uint64_t)IPW(ip, i*64+26) >> 26;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*31+13) |= (uint64_t)IPV(ip, i*64+27) << 5 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*31+14) = (uint64_t)IPW(ip, i*64+28) >> 28;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*31+14) |= (uint64_t)IPV(ip, i*64+29) << 3 | (uint64_t)IPX(ip, i*64+30) << 34;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*31+15) = (uint64_t)IPW(ip, i*64+30) >> 30;\
- IP9(ip, i*64+31, parm); *((uint64_t *)op+i*31+15) |= (uint64_t)IPV(ip, i*64+31) << 1;\
-}
-
-#define BITPACK64_31(ip, op, parm) { \
- BITBLK64_31(ip, 0, op, parm); IPI(ip); op += 31*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_32(ip, i, op, parm) { \
- IP9(ip, i*2+ 0, parm); *(uint32_t *)(op+i*8+ 0) = IPV(ip, i*2+ 0);\
- IP9(ip, i*2+ 1, parm); *(uint32_t *)(op+i*8+ 4) = IPV(ip, i*2+ 1);;\
-}
-
-#define BITPACK64_32(ip, op, parm) { \
- BITBLK64_32(ip, 0, op, parm);\
- BITBLK64_32(ip, 1, op, parm);\
- BITBLK64_32(ip, 2, op, parm);\
- BITBLK64_32(ip, 3, op, parm);\
- BITBLK64_32(ip, 4, op, parm);\
- BITBLK64_32(ip, 5, op, parm);\
- BITBLK64_32(ip, 6, op, parm);\
- BITBLK64_32(ip, 7, op, parm);\
- BITBLK64_32(ip, 8, op, parm);\
- BITBLK64_32(ip, 9, op, parm);\
- BITBLK64_32(ip, 10, op, parm);\
- BITBLK64_32(ip, 11, op, parm);\
- BITBLK64_32(ip, 12, op, parm);\
- BITBLK64_32(ip, 13, op, parm);\
- BITBLK64_32(ip, 14, op, parm);\
- BITBLK64_32(ip, 15, op, parm); IPI(ip); op += 32*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_33(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*33+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 33;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*33+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 31;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*33+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 2 | (uint64_t)IPX(ip, i*64+3) << 35;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*33+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 29;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*33+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 4 | (uint64_t)IPX(ip, i*64+5) << 37;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*33+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 27;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*33+ 3) |= (uint64_t)IPV(ip, i*64+ 6) << 6 | (uint64_t)IPX(ip, i*64+7) << 39;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*33+ 4) = (uint64_t)IPW(ip, i*64+ 7) >> 25;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*33+ 4) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 41;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*33+ 5) = (uint64_t)IPW(ip, i*64+ 9) >> 23;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*33+ 5) |= (uint64_t)IPV(ip, i*64+10) << 10 | (uint64_t)IPX(ip, i*64+11) << 43;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*33+ 6) = (uint64_t)IPW(ip, i*64+11) >> 21;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*33+ 6) |= (uint64_t)IPV(ip, i*64+12) << 12 | (uint64_t)IPX(ip, i*64+13) << 45;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*33+ 7) = (uint64_t)IPW(ip, i*64+13) >> 19;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*33+ 7) |= (uint64_t)IPV(ip, i*64+14) << 14 | (uint64_t)IPX(ip, i*64+15) << 47;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*33+ 8) = (uint64_t)IPW(ip, i*64+15) >> 17;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*33+ 8) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 49;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*33+ 9) = (uint64_t)IPW(ip, i*64+17) >> 15;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*33+ 9) |= (uint64_t)IPV(ip, i*64+18) << 18 | (uint64_t)IPX(ip, i*64+19) << 51;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*33+10) = (uint64_t)IPW(ip, i*64+19) >> 13;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*33+10) |= (uint64_t)IPV(ip, i*64+20) << 20 | (uint64_t)IPX(ip, i*64+21) << 53;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*33+11) = (uint64_t)IPW(ip, i*64+21) >> 11;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*33+11) |= (uint64_t)IPV(ip, i*64+22) << 22 | (uint64_t)IPX(ip, i*64+23) << 55;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*33+12) = (uint64_t)IPW(ip, i*64+23) >> 9;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*33+12) |= (uint64_t)IPV(ip, i*64+24) << 24 | (uint64_t)IPX(ip, i*64+25) << 57;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*33+13) = (uint64_t)IPW(ip, i*64+25) >> 7;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*33+13) |= (uint64_t)IPV(ip, i*64+26) << 26 | (uint64_t)IPX(ip, i*64+27) << 59;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*33+14) = (uint64_t)IPW(ip, i*64+27) >> 5;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*33+14) |= (uint64_t)IPV(ip, i*64+28) << 28 | (uint64_t)IPX(ip, i*64+29) << 61;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*33+15) = (uint64_t)IPW(ip, i*64+29) >> 3;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*33+15) |= (uint64_t)IPV(ip, i*64+30) << 30 | (uint64_t)IPX(ip, i*64+31) << 63;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*33+16) = (uint64_t)IPW(ip, i*64+31) >> 1;\
-}
-
-#define BITPACK64_33(ip, op, parm) { \
- BITBLK64_33(ip, 0, op, parm); IPI(ip); op += 33*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_34(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*17+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 34;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*17+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 30;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*17+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 4 | (uint64_t)IPX(ip, i*32+3) << 38;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*17+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 26;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*17+ 2) |= (uint64_t)IPV(ip, i*32+ 4) << 8 | (uint64_t)IPX(ip, i*32+5) << 42;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*17+ 3) = (uint64_t)IPW(ip, i*32+ 5) >> 22;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*17+ 3) |= (uint64_t)IPV(ip, i*32+ 6) << 12 | (uint64_t)IPX(ip, i*32+7) << 46;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*17+ 4) = (uint64_t)IPW(ip, i*32+ 7) >> 18;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*17+ 4) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 50;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*17+ 5) = (uint64_t)IPW(ip, i*32+ 9) >> 14;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*17+ 5) |= (uint64_t)IPV(ip, i*32+10) << 20 | (uint64_t)IPX(ip, i*32+11) << 54;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*17+ 6) = (uint64_t)IPW(ip, i*32+11) >> 10;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*17+ 6) |= (uint64_t)IPV(ip, i*32+12) << 24 | (uint64_t)IPX(ip, i*32+13) << 58;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*17+ 7) = (uint64_t)IPW(ip, i*32+13) >> 6;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*17+ 7) |= (uint64_t)IPV(ip, i*32+14) << 28 | (uint64_t)IPX(ip, i*32+15) << 62;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*17+ 8) = (uint64_t)IPW(ip, i*32+15) >> 2 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*17+ 9) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*17+ 9) |= (uint64_t)IPV(ip, i*32+17) << 2 | (uint64_t)IPX(ip, i*32+18) << 36;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*17+10) = (uint64_t)IPW(ip, i*32+18) >> 28;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*17+10) |= (uint64_t)IPV(ip, i*32+19) << 6 | (uint64_t)IPX(ip, i*32+20) << 40;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*17+11) = (uint64_t)IPW(ip, i*32+20) >> 24;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*17+11) |= (uint64_t)IPV(ip, i*32+21) << 10 | (uint64_t)IPX(ip, i*32+22) << 44;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*17+12) = (uint64_t)IPW(ip, i*32+22) >> 20;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*17+12) |= (uint64_t)IPV(ip, i*32+23) << 14 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*17+13) = (uint64_t)IPW(ip, i*32+24) >> 16;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*17+13) |= (uint64_t)IPV(ip, i*32+25) << 18 | (uint64_t)IPX(ip, i*32+26) << 52;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*17+14) = (uint64_t)IPW(ip, i*32+26) >> 12;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*17+14) |= (uint64_t)IPV(ip, i*32+27) << 22 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*17+15) = (uint64_t)IPW(ip, i*32+28) >> 8;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*17+15) |= (uint64_t)IPV(ip, i*32+29) << 26 | (uint64_t)IPX(ip, i*32+30) << 60;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*17+16) = (uint64_t)IPW(ip, i*32+30) >> 4;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*17+16) |= (uint64_t)IPV(ip, i*32+31) << 30;\
-}
-
-#define BITPACK64_34(ip, op, parm) { \
- BITBLK64_34(ip, 0, op, parm); IPI(ip); op += 34*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_35(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*35+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 35;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*35+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 29;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*35+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 6 | (uint64_t)IPX(ip, i*64+3) << 41;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*35+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 23;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*35+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 12 | (uint64_t)IPX(ip, i*64+5) << 47;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*35+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 17;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*35+ 3) |= (uint64_t)IPV(ip, i*64+ 6) << 18 | (uint64_t)IPX(ip, i*64+7) << 53;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*35+ 4) = (uint64_t)IPW(ip, i*64+ 7) >> 11;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*35+ 4) |= (uint64_t)IPV(ip, i*64+ 8) << 24 | (uint64_t)IPX(ip, i*64+9) << 59;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*35+ 5) = (uint64_t)IPW(ip, i*64+ 9) >> 5 | (uint64_t)IPX(ip, i*64+10) << 30;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*35+ 6) = (uint64_t)IPW(ip, i*64+10) >> 34;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*35+ 6) |= (uint64_t)IPV(ip, i*64+11) << 1 | (uint64_t)IPX(ip, i*64+12) << 36;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*35+ 7) = (uint64_t)IPW(ip, i*64+12) >> 28;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*35+ 7) |= (uint64_t)IPV(ip, i*64+13) << 7 | (uint64_t)IPX(ip, i*64+14) << 42;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*35+ 8) = (uint64_t)IPW(ip, i*64+14) >> 22;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*35+ 8) |= (uint64_t)IPV(ip, i*64+15) << 13 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*35+ 9) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*35+ 9) |= (uint64_t)IPV(ip, i*64+17) << 19 | (uint64_t)IPX(ip, i*64+18) << 54;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*35+10) = (uint64_t)IPW(ip, i*64+18) >> 10;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*35+10) |= (uint64_t)IPV(ip, i*64+19) << 25 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*35+11) = (uint64_t)IPW(ip, i*64+20) >> 4 | (uint64_t)IPX(ip, i*64+21) << 31;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*35+12) = (uint64_t)IPW(ip, i*64+21) >> 33;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*35+12) |= (uint64_t)IPV(ip, i*64+22) << 2 | (uint64_t)IPX(ip, i*64+23) << 37;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*35+13) = (uint64_t)IPW(ip, i*64+23) >> 27;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*35+13) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 43;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*35+14) = (uint64_t)IPW(ip, i*64+25) >> 21;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*35+14) |= (uint64_t)IPV(ip, i*64+26) << 14 | (uint64_t)IPX(ip, i*64+27) << 49;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*35+15) = (uint64_t)IPW(ip, i*64+27) >> 15;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*35+15) |= (uint64_t)IPV(ip, i*64+28) << 20 | (uint64_t)IPX(ip, i*64+29) << 55;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*35+16) = (uint64_t)IPW(ip, i*64+29) >> 9;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*35+16) |= (uint64_t)IPV(ip, i*64+30) << 26 | (uint64_t)IPX(ip, i*64+31) << 61;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*35+17) = (uint64_t)IPW(ip, i*64+31) >> 3;\
-}
-
-#define BITPACK64_35(ip, op, parm) { \
- BITBLK64_35(ip, 0, op, parm); IPI(ip); op += 35*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_36(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*9+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 36;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*9+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 28;\
- IP9(ip, i*16+ 2, parm); *((uint64_t *)op+i*9+ 1) |= (uint64_t)IPV(ip, i*16+ 2) << 8 | (uint64_t)IPX(ip, i*16+3) << 44;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*9+ 2) = (uint64_t)IPW(ip, i*16+ 3) >> 20;\
- IP9(ip, i*16+ 4, parm); *((uint64_t *)op+i*9+ 2) |= (uint64_t)IPV(ip, i*16+ 4) << 16 | (uint64_t)IPX(ip, i*16+5) << 52;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*9+ 3) = (uint64_t)IPW(ip, i*16+ 5) >> 12;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*9+ 3) |= (uint64_t)IPV(ip, i*16+ 6) << 24 | (uint64_t)IPX(ip, i*16+7) << 60;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*9+ 4) = (uint64_t)IPW(ip, i*16+ 7) >> 4 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*9+ 5) = (uint64_t)IPW(ip, i*16+ 8) >> 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*9+ 5) |= (uint64_t)IPV(ip, i*16+ 9) << 4 | (uint64_t)IPX(ip, i*16+10) << 40;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*9+ 6) = (uint64_t)IPW(ip, i*16+10) >> 24;\
- IP9(ip, i*16+11, parm); *((uint64_t *)op+i*9+ 6) |= (uint64_t)IPV(ip, i*16+11) << 12 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*9+ 7) = (uint64_t)IPW(ip, i*16+12) >> 16;\
- IP9(ip, i*16+13, parm); *((uint64_t *)op+i*9+ 7) |= (uint64_t)IPV(ip, i*16+13) << 20 | (uint64_t)IPX(ip, i*16+14) << 56;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*9+ 8) = (uint64_t)IPW(ip, i*16+14) >> 8;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*9+ 8) |= (uint64_t)IPV(ip, i*16+15) << 28;\
-}
-
-#define BITPACK64_36(ip, op, parm) { \
- BITBLK64_36(ip, 0, op, parm);\
- BITBLK64_36(ip, 1, op, parm); IPI(ip); op += 36*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_37(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*37+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 37;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*37+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 27;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*37+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 10 | (uint64_t)IPX(ip, i*64+3) << 47;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*37+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 17;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*37+ 2) |= (uint64_t)IPV(ip, i*64+ 4) << 20 | (uint64_t)IPX(ip, i*64+5) << 57;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*37+ 3) = (uint64_t)IPW(ip, i*64+ 5) >> 7 | (uint64_t)IPX(ip, i*64+6) << 30;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*37+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 34;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*37+ 4) |= (uint64_t)IPV(ip, i*64+ 7) << 3 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*37+ 5) = (uint64_t)IPW(ip, i*64+ 8) >> 24;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*37+ 5) |= (uint64_t)IPV(ip, i*64+ 9) << 13 | (uint64_t)IPX(ip, i*64+10) << 50;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*37+ 6) = (uint64_t)IPW(ip, i*64+10) >> 14;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*37+ 6) |= (uint64_t)IPV(ip, i*64+11) << 23 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*37+ 7) = (uint64_t)IPW(ip, i*64+12) >> 4 | (uint64_t)IPX(ip, i*64+13) << 33;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*37+ 8) = (uint64_t)IPW(ip, i*64+13) >> 31;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*37+ 8) |= (uint64_t)IPV(ip, i*64+14) << 6 | (uint64_t)IPX(ip, i*64+15) << 43;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*37+ 9) = (uint64_t)IPW(ip, i*64+15) >> 21;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*37+ 9) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 53;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*37+10) = (uint64_t)IPW(ip, i*64+17) >> 11;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*37+10) |= (uint64_t)IPV(ip, i*64+18) << 26 | (uint64_t)IPX(ip, i*64+19) << 63;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*37+11) = (uint64_t)IPW(ip, i*64+19) >> 1 | (uint64_t)IPX(ip, i*64+20) << 36;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*37+12) = (uint64_t)IPW(ip, i*64+20) >> 28;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*37+12) |= (uint64_t)IPV(ip, i*64+21) << 9 | (uint64_t)IPX(ip, i*64+22) << 46;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*37+13) = (uint64_t)IPW(ip, i*64+22) >> 18;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*37+13) |= (uint64_t)IPV(ip, i*64+23) << 19 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*37+14) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 29;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*37+15) = (uint64_t)IPW(ip, i*64+25) >> 35;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*37+15) |= (uint64_t)IPV(ip, i*64+26) << 2 | (uint64_t)IPX(ip, i*64+27) << 39;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*37+16) = (uint64_t)IPW(ip, i*64+27) >> 25;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*37+16) |= (uint64_t)IPV(ip, i*64+28) << 12 | (uint64_t)IPX(ip, i*64+29) << 49;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*37+17) = (uint64_t)IPW(ip, i*64+29) >> 15;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*37+17) |= (uint64_t)IPV(ip, i*64+30) << 22 | (uint64_t)IPX(ip, i*64+31) << 59;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*37+18) = (uint64_t)IPW(ip, i*64+31) >> 5;\
-}
-
-#define BITPACK64_37(ip, op, parm) { \
- BITBLK64_37(ip, 0, op, parm); IPI(ip); op += 37*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_38(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*19+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 38;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*19+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 26;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*19+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 12 | (uint64_t)IPX(ip, i*32+3) << 50;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*19+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 14;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*19+ 2) |= (uint64_t)IPV(ip, i*32+ 4) << 24 | (uint64_t)IPX(ip, i*32+5) << 62;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*19+ 3) = (uint64_t)IPW(ip, i*32+ 5) >> 2 | (uint64_t)IPX(ip, i*32+6) << 36;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*19+ 4) = (uint64_t)IPW(ip, i*32+ 6) >> 28;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*19+ 4) |= (uint64_t)IPV(ip, i*32+ 7) << 10 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*19+ 5) = (uint64_t)IPW(ip, i*32+ 8) >> 16;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*19+ 5) |= (uint64_t)IPV(ip, i*32+ 9) << 22 | (uint64_t)IPX(ip, i*32+10) << 60;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*19+ 6) = (uint64_t)IPW(ip, i*32+10) >> 4 | (uint64_t)IPX(ip, i*32+11) << 34;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*19+ 7) = (uint64_t)IPW(ip, i*32+11) >> 30;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*19+ 7) |= (uint64_t)IPV(ip, i*32+12) << 8 | (uint64_t)IPX(ip, i*32+13) << 46;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*19+ 8) = (uint64_t)IPW(ip, i*32+13) >> 18;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*19+ 8) |= (uint64_t)IPV(ip, i*32+14) << 20 | (uint64_t)IPX(ip, i*32+15) << 58;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*19+ 9) = (uint64_t)IPW(ip, i*32+15) >> 6 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*19+10) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*19+10) |= (uint64_t)IPV(ip, i*32+17) << 6 | (uint64_t)IPX(ip, i*32+18) << 44;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*19+11) = (uint64_t)IPW(ip, i*32+18) >> 20;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*19+11) |= (uint64_t)IPV(ip, i*32+19) << 18 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*19+12) = (uint64_t)IPW(ip, i*32+20) >> 8 | (uint64_t)IPX(ip, i*32+21) << 30;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*19+13) = (uint64_t)IPW(ip, i*32+21) >> 34;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*19+13) |= (uint64_t)IPV(ip, i*32+22) << 4 | (uint64_t)IPX(ip, i*32+23) << 42;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*19+14) = (uint64_t)IPW(ip, i*32+23) >> 22;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*19+14) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 54;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*19+15) = (uint64_t)IPW(ip, i*32+25) >> 10 | (uint64_t)IPX(ip, i*32+26) << 28;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*19+16) = (uint64_t)IPW(ip, i*32+26) >> 36;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*19+16) |= (uint64_t)IPV(ip, i*32+27) << 2 | (uint64_t)IPX(ip, i*32+28) << 40;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*19+17) = (uint64_t)IPW(ip, i*32+28) >> 24;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*19+17) |= (uint64_t)IPV(ip, i*32+29) << 14 | (uint64_t)IPX(ip, i*32+30) << 52;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*19+18) = (uint64_t)IPW(ip, i*32+30) >> 12;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*19+18) |= (uint64_t)IPV(ip, i*32+31) << 26;\
-}
-
-#define BITPACK64_38(ip, op, parm) { \
- BITBLK64_38(ip, 0, op, parm); IPI(ip); op += 38*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_39(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*39+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 39;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*39+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 25;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*39+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 14 | (uint64_t)IPX(ip, i*64+3) << 53;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*39+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 11 | (uint64_t)IPX(ip, i*64+4) << 28;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*39+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 36;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*39+ 3) |= (uint64_t)IPV(ip, i*64+ 5) << 3 | (uint64_t)IPX(ip, i*64+6) << 42;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*39+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 22;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*39+ 4) |= (uint64_t)IPV(ip, i*64+ 7) << 17 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*39+ 5) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 31;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*39+ 6) = (uint64_t)IPW(ip, i*64+ 9) >> 33;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*39+ 6) |= (uint64_t)IPV(ip, i*64+10) << 6 | (uint64_t)IPX(ip, i*64+11) << 45;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*39+ 7) = (uint64_t)IPW(ip, i*64+11) >> 19;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*39+ 7) |= (uint64_t)IPV(ip, i*64+12) << 20 | (uint64_t)IPX(ip, i*64+13) << 59;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*39+ 8) = (uint64_t)IPW(ip, i*64+13) >> 5 | (uint64_t)IPX(ip, i*64+14) << 34;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*39+ 9) = (uint64_t)IPW(ip, i*64+14) >> 30;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*39+ 9) |= (uint64_t)IPV(ip, i*64+15) << 9 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*39+10) = (uint64_t)IPW(ip, i*64+16) >> 16;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*39+10) |= (uint64_t)IPV(ip, i*64+17) << 23 | (uint64_t)IPX(ip, i*64+18) << 62;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*39+11) = (uint64_t)IPW(ip, i*64+18) >> 2 | (uint64_t)IPX(ip, i*64+19) << 37;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*39+12) = (uint64_t)IPW(ip, i*64+19) >> 27;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*39+12) |= (uint64_t)IPV(ip, i*64+20) << 12 | (uint64_t)IPX(ip, i*64+21) << 51;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*39+13) = (uint64_t)IPW(ip, i*64+21) >> 13 | (uint64_t)IPX(ip, i*64+22) << 26;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*39+14) = (uint64_t)IPW(ip, i*64+22) >> 38;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*39+14) |= (uint64_t)IPV(ip, i*64+23) << 1 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*39+15) = (uint64_t)IPW(ip, i*64+24) >> 24;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*39+15) |= (uint64_t)IPV(ip, i*64+25) << 15 | (uint64_t)IPX(ip, i*64+26) << 54;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*39+16) = (uint64_t)IPW(ip, i*64+26) >> 10 | (uint64_t)IPX(ip, i*64+27) << 29;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*39+17) = (uint64_t)IPW(ip, i*64+27) >> 35;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*39+17) |= (uint64_t)IPV(ip, i*64+28) << 4 | (uint64_t)IPX(ip, i*64+29) << 43;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*39+18) = (uint64_t)IPW(ip, i*64+29) >> 21;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*39+18) |= (uint64_t)IPV(ip, i*64+30) << 18 | (uint64_t)IPX(ip, i*64+31) << 57;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*39+19) = (uint64_t)IPW(ip, i*64+31) >> 7;\
-}
-
-#define BITPACK64_39(ip, op, parm) { \
- BITBLK64_39(ip, 0, op, parm); IPI(ip); op += 39*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_40(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*5+ 0) = (uint64_t)IPV(ip, i*8+ 0) | (uint64_t)IPX(ip, i*8+1) << 40;\
- IP64(ip, i*8+ 1, parm); *((uint64_t *)op+i*5+ 1) = (uint64_t)IPW(ip, i*8+ 1) >> 24;\
- IP9(ip, i*8+ 2, parm); *((uint64_t *)op+i*5+ 1) |= (uint64_t)IPV(ip, i*8+ 2) << 16 | (uint64_t)IPX(ip, i*8+3) << 56;\
- IP64(ip, i*8+ 3, parm); *((uint64_t *)op+i*5+ 2) = (uint64_t)IPW(ip, i*8+ 3) >> 8 | (uint64_t)IPX(ip, i*8+4) << 32;\
- IP64(ip, i*8+ 4, parm); *((uint64_t *)op+i*5+ 3) = (uint64_t)IPW(ip, i*8+ 4) >> 32;\
- IP9(ip, i*8+ 5, parm); *((uint64_t *)op+i*5+ 3) |= (uint64_t)IPV(ip, i*8+ 5) << 8 | (uint64_t)IPX(ip, i*8+6) << 48;\
- IP64(ip, i*8+ 6, parm); *((uint64_t *)op+i*5+ 4) = (uint64_t)IPW(ip, i*8+ 6) >> 16;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*5+ 4) |= (uint64_t)IPV(ip, i*8+ 7) << 24;\
-}
-
-#define BITPACK64_40(ip, op, parm) { \
- BITBLK64_40(ip, 0, op, parm);\
- BITBLK64_40(ip, 1, op, parm);\
- BITBLK64_40(ip, 2, op, parm);\
- BITBLK64_40(ip, 3, op, parm); IPI(ip); op += 40*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_41(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*41+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 41;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*41+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 23;\
- IP9(ip, i*64+ 2, parm); *((uint64_t *)op+i*41+ 1) |= (uint64_t)IPV(ip, i*64+ 2) << 18 | (uint64_t)IPX(ip, i*64+3) << 59;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*41+ 2) = (uint64_t)IPW(ip, i*64+ 3) >> 5 | (uint64_t)IPX(ip, i*64+4) << 36;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*41+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 28;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*41+ 3) |= (uint64_t)IPV(ip, i*64+ 5) << 13 | (uint64_t)IPX(ip, i*64+6) << 54;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*41+ 4) = (uint64_t)IPW(ip, i*64+ 6) >> 10 | (uint64_t)IPX(ip, i*64+7) << 31;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*41+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 33;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*41+ 5) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 49;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*41+ 6) = (uint64_t)IPW(ip, i*64+ 9) >> 15 | (uint64_t)IPX(ip, i*64+10) << 26;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*41+ 7) = (uint64_t)IPW(ip, i*64+10) >> 38;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*41+ 7) |= (uint64_t)IPV(ip, i*64+11) << 3 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*41+ 8) = (uint64_t)IPW(ip, i*64+12) >> 20;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*41+ 8) |= (uint64_t)IPV(ip, i*64+13) << 21 | (uint64_t)IPX(ip, i*64+14) << 62;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*41+ 9) = (uint64_t)IPW(ip, i*64+14) >> 2 | (uint64_t)IPX(ip, i*64+15) << 39;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*41+10) = (uint64_t)IPW(ip, i*64+15) >> 25;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*41+10) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 57;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*41+11) = (uint64_t)IPW(ip, i*64+17) >> 7 | (uint64_t)IPX(ip, i*64+18) << 34;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*41+12) = (uint64_t)IPW(ip, i*64+18) >> 30;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*41+12) |= (uint64_t)IPV(ip, i*64+19) << 11 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*41+13) = (uint64_t)IPW(ip, i*64+20) >> 12 | (uint64_t)IPX(ip, i*64+21) << 29;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*41+14) = (uint64_t)IPW(ip, i*64+21) >> 35;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*41+14) |= (uint64_t)IPV(ip, i*64+22) << 6 | (uint64_t)IPX(ip, i*64+23) << 47;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*41+15) = (uint64_t)IPW(ip, i*64+23) >> 17 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*41+16) = (uint64_t)IPW(ip, i*64+24) >> 40;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*41+16) |= (uint64_t)IPV(ip, i*64+25) << 1 | (uint64_t)IPX(ip, i*64+26) << 42;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*41+17) = (uint64_t)IPW(ip, i*64+26) >> 22;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*41+17) |= (uint64_t)IPV(ip, i*64+27) << 19 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*41+18) = (uint64_t)IPW(ip, i*64+28) >> 4 | (uint64_t)IPX(ip, i*64+29) << 37;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*41+19) = (uint64_t)IPW(ip, i*64+29) >> 27;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*41+19) |= (uint64_t)IPV(ip, i*64+30) << 14 | (uint64_t)IPX(ip, i*64+31) << 55;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*41+20) = (uint64_t)IPW(ip, i*64+31) >> 9;\
-}
-
-#define BITPACK64_41(ip, op, parm) { \
- BITBLK64_41(ip, 0, op, parm); IPI(ip); op += 41*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_42(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*21+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 42;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*21+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 22;\
- IP9(ip, i*32+ 2, parm); *((uint64_t *)op+i*21+ 1) |= (uint64_t)IPV(ip, i*32+ 2) << 20 | (uint64_t)IPX(ip, i*32+3) << 62;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*21+ 2) = (uint64_t)IPW(ip, i*32+ 3) >> 2 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*21+ 3) = (uint64_t)IPW(ip, i*32+ 4) >> 24;\
- IP9(ip, i*32+ 5, parm); *((uint64_t *)op+i*21+ 3) |= (uint64_t)IPV(ip, i*32+ 5) << 18 | (uint64_t)IPX(ip, i*32+6) << 60;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*21+ 4) = (uint64_t)IPW(ip, i*32+ 6) >> 4 | (uint64_t)IPX(ip, i*32+7) << 38;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*21+ 5) = (uint64_t)IPW(ip, i*32+ 7) >> 26;\
- IP9(ip, i*32+ 8, parm); *((uint64_t *)op+i*21+ 5) |= (uint64_t)IPV(ip, i*32+ 8) << 16 | (uint64_t)IPX(ip, i*32+9) << 58;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*21+ 6) = (uint64_t)IPW(ip, i*32+ 9) >> 6 | (uint64_t)IPX(ip, i*32+10) << 36;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*21+ 7) = (uint64_t)IPW(ip, i*32+10) >> 28;\
- IP9(ip, i*32+11, parm); *((uint64_t *)op+i*21+ 7) |= (uint64_t)IPV(ip, i*32+11) << 14 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*21+ 8) = (uint64_t)IPW(ip, i*32+12) >> 8 | (uint64_t)IPX(ip, i*32+13) << 34;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*21+ 9) = (uint64_t)IPW(ip, i*32+13) >> 30;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*21+ 9) |= (uint64_t)IPV(ip, i*32+14) << 12 | (uint64_t)IPX(ip, i*32+15) << 54;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*21+10) = (uint64_t)IPW(ip, i*32+15) >> 10 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*21+11) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*21+11) |= (uint64_t)IPV(ip, i*32+17) << 10 | (uint64_t)IPX(ip, i*32+18) << 52;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*21+12) = (uint64_t)IPW(ip, i*32+18) >> 12 | (uint64_t)IPX(ip, i*32+19) << 30;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*21+13) = (uint64_t)IPW(ip, i*32+19) >> 34;\
- IP9(ip, i*32+20, parm); *((uint64_t *)op+i*21+13) |= (uint64_t)IPV(ip, i*32+20) << 8 | (uint64_t)IPX(ip, i*32+21) << 50;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*21+14) = (uint64_t)IPW(ip, i*32+21) >> 14 | (uint64_t)IPX(ip, i*32+22) << 28;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*21+15) = (uint64_t)IPW(ip, i*32+22) >> 36;\
- IP9(ip, i*32+23, parm); *((uint64_t *)op+i*21+15) |= (uint64_t)IPV(ip, i*32+23) << 6 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*21+16) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 26;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*21+17) = (uint64_t)IPW(ip, i*32+25) >> 38;\
- IP9(ip, i*32+26, parm); *((uint64_t *)op+i*21+17) |= (uint64_t)IPV(ip, i*32+26) << 4 | (uint64_t)IPX(ip, i*32+27) << 46;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*21+18) = (uint64_t)IPW(ip, i*32+27) >> 18 | (uint64_t)IPX(ip, i*32+28) << 24;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*21+19) = (uint64_t)IPW(ip, i*32+28) >> 40;\
- IP9(ip, i*32+29, parm); *((uint64_t *)op+i*21+19) |= (uint64_t)IPV(ip, i*32+29) << 2 | (uint64_t)IPX(ip, i*32+30) << 44;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*21+20) = (uint64_t)IPW(ip, i*32+30) >> 20;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*21+20) |= (uint64_t)IPV(ip, i*32+31) << 22;\
-}
-
-#define BITPACK64_42(ip, op, parm) { \
- BITBLK64_42(ip, 0, op, parm); IPI(ip); op += 42*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_43(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*43+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 43;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*43+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 21 | (uint64_t)IPX(ip, i*64+2) << 22;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*43+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 42;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*43+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 1 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*43+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 20 | (uint64_t)IPX(ip, i*64+5) << 23;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*43+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 41;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*43+ 4) |= (uint64_t)IPV(ip, i*64+ 6) << 2 | (uint64_t)IPX(ip, i*64+7) << 45;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*43+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 19 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*43+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 40;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*43+ 6) |= (uint64_t)IPV(ip, i*64+ 9) << 3 | (uint64_t)IPX(ip, i*64+10) << 46;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*43+ 7) = (uint64_t)IPW(ip, i*64+10) >> 18 | (uint64_t)IPX(ip, i*64+11) << 25;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*43+ 8) = (uint64_t)IPW(ip, i*64+11) >> 39;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*43+ 8) |= (uint64_t)IPV(ip, i*64+12) << 4 | (uint64_t)IPX(ip, i*64+13) << 47;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*43+ 9) = (uint64_t)IPW(ip, i*64+13) >> 17 | (uint64_t)IPX(ip, i*64+14) << 26;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*43+10) = (uint64_t)IPW(ip, i*64+14) >> 38;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*43+10) |= (uint64_t)IPV(ip, i*64+15) << 5 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*43+11) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 27;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*43+12) = (uint64_t)IPW(ip, i*64+17) >> 37;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*43+12) |= (uint64_t)IPV(ip, i*64+18) << 6 | (uint64_t)IPX(ip, i*64+19) << 49;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*43+13) = (uint64_t)IPW(ip, i*64+19) >> 15 | (uint64_t)IPX(ip, i*64+20) << 28;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*43+14) = (uint64_t)IPW(ip, i*64+20) >> 36;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*43+14) |= (uint64_t)IPV(ip, i*64+21) << 7 | (uint64_t)IPX(ip, i*64+22) << 50;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*43+15) = (uint64_t)IPW(ip, i*64+22) >> 14 | (uint64_t)IPX(ip, i*64+23) << 29;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*43+16) = (uint64_t)IPW(ip, i*64+23) >> 35;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*43+16) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 51;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*43+17) = (uint64_t)IPW(ip, i*64+25) >> 13 | (uint64_t)IPX(ip, i*64+26) << 30;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*43+18) = (uint64_t)IPW(ip, i*64+26) >> 34;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*43+18) |= (uint64_t)IPV(ip, i*64+27) << 9 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*43+19) = (uint64_t)IPW(ip, i*64+28) >> 12 | (uint64_t)IPX(ip, i*64+29) << 31;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*43+20) = (uint64_t)IPW(ip, i*64+29) >> 33;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*43+20) |= (uint64_t)IPV(ip, i*64+30) << 10 | (uint64_t)IPX(ip, i*64+31) << 53;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*43+21) = (uint64_t)IPW(ip, i*64+31) >> 11;\
-}
-
-#define BITPACK64_43(ip, op, parm) { \
- BITBLK64_43(ip, 0, op, parm); IPI(ip); op += 43*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_44(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*11+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 44;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*11+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 20 | (uint64_t)IPX(ip, i*16+2) << 24;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*11+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 40;\
- IP9(ip, i*16+ 3, parm); *((uint64_t *)op+i*11+ 2) |= (uint64_t)IPV(ip, i*16+ 3) << 4 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*11+ 3) = (uint64_t)IPW(ip, i*16+ 4) >> 16 | (uint64_t)IPX(ip, i*16+5) << 28;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*11+ 4) = (uint64_t)IPW(ip, i*16+ 5) >> 36;\
- IP9(ip, i*16+ 6, parm); *((uint64_t *)op+i*11+ 4) |= (uint64_t)IPV(ip, i*16+ 6) << 8 | (uint64_t)IPX(ip, i*16+7) << 52;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*11+ 5) = (uint64_t)IPW(ip, i*16+ 7) >> 12 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*11+ 6) = (uint64_t)IPW(ip, i*16+ 8) >> 32;\
- IP9(ip, i*16+ 9, parm); *((uint64_t *)op+i*11+ 6) |= (uint64_t)IPV(ip, i*16+ 9) << 12 | (uint64_t)IPX(ip, i*16+10) << 56;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*11+ 7) = (uint64_t)IPW(ip, i*16+10) >> 8 | (uint64_t)IPX(ip, i*16+11) << 36;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*11+ 8) = (uint64_t)IPW(ip, i*16+11) >> 28;\
- IP9(ip, i*16+12, parm); *((uint64_t *)op+i*11+ 8) |= (uint64_t)IPV(ip, i*16+12) << 16 | (uint64_t)IPX(ip, i*16+13) << 60;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*11+ 9) = (uint64_t)IPW(ip, i*16+13) >> 4 | (uint64_t)IPX(ip, i*16+14) << 40;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*11+10) = (uint64_t)IPW(ip, i*16+14) >> 24;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*11+10) |= (uint64_t)IPV(ip, i*16+15) << 20;\
-}
-
-#define BITPACK64_44(ip, op, parm) { \
- BITBLK64_44(ip, 0, op, parm);\
- BITBLK64_44(ip, 1, op, parm); IPI(ip); op += 44*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_45(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*45+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 45;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*45+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 19 | (uint64_t)IPX(ip, i*64+2) << 26;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*45+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 38;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*45+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 7 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*45+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 12 | (uint64_t)IPX(ip, i*64+5) << 33;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*45+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 31;\
- IP9(ip, i*64+ 6, parm); *((uint64_t *)op+i*45+ 4) |= (uint64_t)IPV(ip, i*64+ 6) << 14 | (uint64_t)IPX(ip, i*64+7) << 59;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*45+ 5) = (uint64_t)IPW(ip, i*64+ 7) >> 5 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*45+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 21;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*45+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 43;\
- IP9(ip, i*64+10, parm); *((uint64_t *)op+i*45+ 7) |= (uint64_t)IPV(ip, i*64+10) << 2 | (uint64_t)IPX(ip, i*64+11) << 47;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*45+ 8) = (uint64_t)IPW(ip, i*64+11) >> 17 | (uint64_t)IPX(ip, i*64+12) << 28;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*45+ 9) = (uint64_t)IPW(ip, i*64+12) >> 36;\
- IP9(ip, i*64+13, parm); *((uint64_t *)op+i*45+ 9) |= (uint64_t)IPV(ip, i*64+13) << 9 | (uint64_t)IPX(ip, i*64+14) << 54;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*45+10) = (uint64_t)IPW(ip, i*64+14) >> 10 | (uint64_t)IPX(ip, i*64+15) << 35;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*45+11) = (uint64_t)IPW(ip, i*64+15) >> 29;\
- IP9(ip, i*64+16, parm); *((uint64_t *)op+i*45+11) |= (uint64_t)IPV(ip, i*64+16) << 16 | (uint64_t)IPX(ip, i*64+17) << 61;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*45+12) = (uint64_t)IPW(ip, i*64+17) >> 3 | (uint64_t)IPX(ip, i*64+18) << 42;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*45+13) = (uint64_t)IPW(ip, i*64+18) >> 22 | (uint64_t)IPX(ip, i*64+19) << 23;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*45+14) = (uint64_t)IPW(ip, i*64+19) >> 41;\
- IP9(ip, i*64+20, parm); *((uint64_t *)op+i*45+14) |= (uint64_t)IPV(ip, i*64+20) << 4 | (uint64_t)IPX(ip, i*64+21) << 49;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*45+15) = (uint64_t)IPW(ip, i*64+21) >> 15 | (uint64_t)IPX(ip, i*64+22) << 30;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*45+16) = (uint64_t)IPW(ip, i*64+22) >> 34;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*45+16) |= (uint64_t)IPV(ip, i*64+23) << 11 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*45+17) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 37;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*45+18) = (uint64_t)IPW(ip, i*64+25) >> 27;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*45+18) |= (uint64_t)IPV(ip, i*64+26) << 18 | (uint64_t)IPX(ip, i*64+27) << 63;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*45+19) = (uint64_t)IPW(ip, i*64+27) >> 1 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*45+20) = (uint64_t)IPW(ip, i*64+28) >> 20 | (uint64_t)IPX(ip, i*64+29) << 25;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*45+21) = (uint64_t)IPW(ip, i*64+29) >> 39;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*45+21) |= (uint64_t)IPV(ip, i*64+30) << 6 | (uint64_t)IPX(ip, i*64+31) << 51;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*45+22) = (uint64_t)IPW(ip, i*64+31) >> 13;\
-}
-
-#define BITPACK64_45(ip, op, parm) { \
- BITBLK64_45(ip, 0, op, parm); IPI(ip); op += 45*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_46(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*23+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 46;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*23+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 18 | (uint64_t)IPX(ip, i*32+2) << 28;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*23+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 36;\
- IP9(ip, i*32+ 3, parm); *((uint64_t *)op+i*23+ 2) |= (uint64_t)IPV(ip, i*32+ 3) << 10 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*23+ 3) = (uint64_t)IPW(ip, i*32+ 4) >> 8 | (uint64_t)IPX(ip, i*32+5) << 38;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*23+ 4) = (uint64_t)IPW(ip, i*32+ 5) >> 26 | (uint64_t)IPX(ip, i*32+6) << 20;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*23+ 5) = (uint64_t)IPW(ip, i*32+ 6) >> 44;\
- IP9(ip, i*32+ 7, parm); *((uint64_t *)op+i*23+ 5) |= (uint64_t)IPV(ip, i*32+ 7) << 2 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*23+ 6) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 30;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*23+ 7) = (uint64_t)IPW(ip, i*32+ 9) >> 34;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*23+ 7) |= (uint64_t)IPV(ip, i*32+10) << 12 | (uint64_t)IPX(ip, i*32+11) << 58;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*23+ 8) = (uint64_t)IPW(ip, i*32+11) >> 6 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*23+ 9) = (uint64_t)IPW(ip, i*32+12) >> 24 | (uint64_t)IPX(ip, i*32+13) << 22;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*23+10) = (uint64_t)IPW(ip, i*32+13) >> 42;\
- IP9(ip, i*32+14, parm); *((uint64_t *)op+i*23+10) |= (uint64_t)IPV(ip, i*32+14) << 4 | (uint64_t)IPX(ip, i*32+15) << 50;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*23+11) = (uint64_t)IPW(ip, i*32+15) >> 14 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*23+12) = (uint64_t)IPW(ip, i*32+16) >> 32;\
- IP9(ip, i*32+17, parm); *((uint64_t *)op+i*23+12) |= (uint64_t)IPV(ip, i*32+17) << 14 | (uint64_t)IPX(ip, i*32+18) << 60;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*23+13) = (uint64_t)IPW(ip, i*32+18) >> 4 | (uint64_t)IPX(ip, i*32+19) << 42;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*23+14) = (uint64_t)IPW(ip, i*32+19) >> 22 | (uint64_t)IPX(ip, i*32+20) << 24;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*23+15) = (uint64_t)IPW(ip, i*32+20) >> 40;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*23+15) |= (uint64_t)IPV(ip, i*32+21) << 6 | (uint64_t)IPX(ip, i*32+22) << 52;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*23+16) = (uint64_t)IPW(ip, i*32+22) >> 12 | (uint64_t)IPX(ip, i*32+23) << 34;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*23+17) = (uint64_t)IPW(ip, i*32+23) >> 30;\
- IP9(ip, i*32+24, parm); *((uint64_t *)op+i*23+17) |= (uint64_t)IPV(ip, i*32+24) << 16 | (uint64_t)IPX(ip, i*32+25) << 62;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*23+18) = (uint64_t)IPW(ip, i*32+25) >> 2 | (uint64_t)IPX(ip, i*32+26) << 44;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*23+19) = (uint64_t)IPW(ip, i*32+26) >> 20 | (uint64_t)IPX(ip, i*32+27) << 26;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*23+20) = (uint64_t)IPW(ip, i*32+27) >> 38;\
- IP9(ip, i*32+28, parm); *((uint64_t *)op+i*23+20) |= (uint64_t)IPV(ip, i*32+28) << 8 | (uint64_t)IPX(ip, i*32+29) << 54;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*23+21) = (uint64_t)IPW(ip, i*32+29) >> 10 | (uint64_t)IPX(ip, i*32+30) << 36;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*23+22) = (uint64_t)IPW(ip, i*32+30) >> 28;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*23+22) |= (uint64_t)IPV(ip, i*32+31) << 18;\
-}
-
-#define BITPACK64_46(ip, op, parm) { \
- BITBLK64_46(ip, 0, op, parm); IPI(ip); op += 46*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_47(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*47+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 47;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*47+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 17 | (uint64_t)IPX(ip, i*64+2) << 30;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*47+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 34;\
- IP9(ip, i*64+ 3, parm); *((uint64_t *)op+i*47+ 2) |= (uint64_t)IPV(ip, i*64+ 3) << 13 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*47+ 3) = (uint64_t)IPW(ip, i*64+ 4) >> 4 | (uint64_t)IPX(ip, i*64+5) << 43;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*47+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 21 | (uint64_t)IPX(ip, i*64+6) << 26;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*47+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 38;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*47+ 5) |= (uint64_t)IPV(ip, i*64+ 7) << 9 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*47+ 6) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 39;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*47+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 25 | (uint64_t)IPX(ip, i*64+10) << 22;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*47+ 8) = (uint64_t)IPW(ip, i*64+10) >> 42;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*47+ 8) |= (uint64_t)IPV(ip, i*64+11) << 5 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*47+ 9) = (uint64_t)IPW(ip, i*64+12) >> 12 | (uint64_t)IPX(ip, i*64+13) << 35;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*47+10) = (uint64_t)IPW(ip, i*64+13) >> 29 | (uint64_t)IPX(ip, i*64+14) << 18;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*47+11) = (uint64_t)IPW(ip, i*64+14) >> 46;\
- IP9(ip, i*64+15, parm); *((uint64_t *)op+i*47+11) |= (uint64_t)IPV(ip, i*64+15) << 1 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*47+12) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 31;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*47+13) = (uint64_t)IPW(ip, i*64+17) >> 33;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*47+13) |= (uint64_t)IPV(ip, i*64+18) << 14 | (uint64_t)IPX(ip, i*64+19) << 61;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*47+14) = (uint64_t)IPW(ip, i*64+19) >> 3 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*47+15) = (uint64_t)IPW(ip, i*64+20) >> 20 | (uint64_t)IPX(ip, i*64+21) << 27;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*47+16) = (uint64_t)IPW(ip, i*64+21) >> 37;\
- IP9(ip, i*64+22, parm); *((uint64_t *)op+i*47+16) |= (uint64_t)IPV(ip, i*64+22) << 10 | (uint64_t)IPX(ip, i*64+23) << 57;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*47+17) = (uint64_t)IPW(ip, i*64+23) >> 7 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*47+18) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 23;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*47+19) = (uint64_t)IPW(ip, i*64+25) >> 41;\
- IP9(ip, i*64+26, parm); *((uint64_t *)op+i*47+19) |= (uint64_t)IPV(ip, i*64+26) << 6 | (uint64_t)IPX(ip, i*64+27) << 53;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*47+20) = (uint64_t)IPW(ip, i*64+27) >> 11 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*47+21) = (uint64_t)IPW(ip, i*64+28) >> 28 | (uint64_t)IPX(ip, i*64+29) << 19;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*47+22) = (uint64_t)IPW(ip, i*64+29) >> 45;\
- IP9(ip, i*64+30, parm); *((uint64_t *)op+i*47+22) |= (uint64_t)IPV(ip, i*64+30) << 2 | (uint64_t)IPX(ip, i*64+31) << 49;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*47+23) = (uint64_t)IPW(ip, i*64+31) >> 15;\
-}
-
-#define BITPACK64_47(ip, op, parm) { \
- BITBLK64_47(ip, 0, op, parm); IPI(ip); op += 47*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_48(ip, i, op, parm) { ;\
- IP9(ip, i*4+ 0, parm); *((uint64_t *)op+i*3+ 0) = (uint64_t)IPV(ip, i*4+ 0) | (uint64_t)IPX(ip, i*4+1) << 48;\
- IP64(ip, i*4+ 1, parm); *((uint64_t *)op+i*3+ 1) = (uint64_t)IPW(ip, i*4+ 1) >> 16 | (uint64_t)IPX(ip, i*4+2) << 32;\
- IP64(ip, i*4+ 2, parm); *((uint64_t *)op+i*3+ 2) = (uint64_t)IPW(ip, i*4+ 2) >> 32;\
- IP9(ip, i*4+ 3, parm); *((uint64_t *)op+i*3+ 2) |= (uint64_t)IPV(ip, i*4+ 3) << 16;\
-}
-
-#define BITPACK64_48(ip, op, parm) { \
- BITBLK64_48(ip, 0, op, parm);\
- BITBLK64_48(ip, 1, op, parm);\
- BITBLK64_48(ip, 2, op, parm);\
- BITBLK64_48(ip, 3, op, parm);\
- BITBLK64_48(ip, 4, op, parm);\
- BITBLK64_48(ip, 5, op, parm);\
- BITBLK64_48(ip, 6, op, parm);\
- BITBLK64_48(ip, 7, op, parm); IPI(ip); op += 48*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_49(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*49+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 49;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*49+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 15 | (uint64_t)IPX(ip, i*64+2) << 34;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*49+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 30 | (uint64_t)IPX(ip, i*64+3) << 19;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*49+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 45;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*49+ 3) |= (uint64_t)IPV(ip, i*64+ 4) << 4 | (uint64_t)IPX(ip, i*64+5) << 53;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*49+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 11 | (uint64_t)IPX(ip, i*64+6) << 38;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*49+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 26 | (uint64_t)IPX(ip, i*64+7) << 23;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*49+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 41;\
- IP9(ip, i*64+ 8, parm); *((uint64_t *)op+i*49+ 6) |= (uint64_t)IPV(ip, i*64+ 8) << 8 | (uint64_t)IPX(ip, i*64+9) << 57;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*49+ 7) = (uint64_t)IPW(ip, i*64+ 9) >> 7 | (uint64_t)IPX(ip, i*64+10) << 42;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*49+ 8) = (uint64_t)IPW(ip, i*64+10) >> 22 | (uint64_t)IPX(ip, i*64+11) << 27;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*49+ 9) = (uint64_t)IPW(ip, i*64+11) >> 37;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*49+ 9) |= (uint64_t)IPV(ip, i*64+12) << 12 | (uint64_t)IPX(ip, i*64+13) << 61;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*49+10) = (uint64_t)IPW(ip, i*64+13) >> 3 | (uint64_t)IPX(ip, i*64+14) << 46;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*49+11) = (uint64_t)IPW(ip, i*64+14) >> 18 | (uint64_t)IPX(ip, i*64+15) << 31;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*49+12) = (uint64_t)IPW(ip, i*64+15) >> 33 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*49+13) = (uint64_t)IPW(ip, i*64+16) >> 48;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*49+13) |= (uint64_t)IPV(ip, i*64+17) << 1 | (uint64_t)IPX(ip, i*64+18) << 50;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*49+14) = (uint64_t)IPW(ip, i*64+18) >> 14 | (uint64_t)IPX(ip, i*64+19) << 35;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*49+15) = (uint64_t)IPW(ip, i*64+19) >> 29 | (uint64_t)IPX(ip, i*64+20) << 20;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*49+16) = (uint64_t)IPW(ip, i*64+20) >> 44;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*49+16) |= (uint64_t)IPV(ip, i*64+21) << 5 | (uint64_t)IPX(ip, i*64+22) << 54;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*49+17) = (uint64_t)IPW(ip, i*64+22) >> 10 | (uint64_t)IPX(ip, i*64+23) << 39;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*49+18) = (uint64_t)IPW(ip, i*64+23) >> 25 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*49+19) = (uint64_t)IPW(ip, i*64+24) >> 40;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*49+19) |= (uint64_t)IPV(ip, i*64+25) << 9 | (uint64_t)IPX(ip, i*64+26) << 58;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*49+20) = (uint64_t)IPW(ip, i*64+26) >> 6 | (uint64_t)IPX(ip, i*64+27) << 43;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*49+21) = (uint64_t)IPW(ip, i*64+27) >> 21 | (uint64_t)IPX(ip, i*64+28) << 28;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*49+22) = (uint64_t)IPW(ip, i*64+28) >> 36;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*49+22) |= (uint64_t)IPV(ip, i*64+29) << 13 | (uint64_t)IPX(ip, i*64+30) << 62;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*49+23) = (uint64_t)IPW(ip, i*64+30) >> 2 | (uint64_t)IPX(ip, i*64+31) << 47;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*49+24) = (uint64_t)IPW(ip, i*64+31) >> 17;\
-}
-
-#define BITPACK64_49(ip, op, parm) { \
- BITBLK64_49(ip, 0, op, parm); IPI(ip); op += 49*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_50(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*25+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 50;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*25+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 14 | (uint64_t)IPX(ip, i*32+2) << 36;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*25+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 28 | (uint64_t)IPX(ip, i*32+3) << 22;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*25+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 42;\
- IP9(ip, i*32+ 4, parm); *((uint64_t *)op+i*25+ 3) |= (uint64_t)IPV(ip, i*32+ 4) << 8 | (uint64_t)IPX(ip, i*32+5) << 58;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*25+ 4) = (uint64_t)IPW(ip, i*32+ 5) >> 6 | (uint64_t)IPX(ip, i*32+6) << 44;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*25+ 5) = (uint64_t)IPW(ip, i*32+ 6) >> 20 | (uint64_t)IPX(ip, i*32+7) << 30;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*25+ 6) = (uint64_t)IPW(ip, i*32+ 7) >> 34 | (uint64_t)IPX(ip, i*32+8) << 16;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*25+ 7) = (uint64_t)IPW(ip, i*32+ 8) >> 48;\
- IP9(ip, i*32+ 9, parm); *((uint64_t *)op+i*25+ 7) |= (uint64_t)IPV(ip, i*32+ 9) << 2 | (uint64_t)IPX(ip, i*32+10) << 52;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*25+ 8) = (uint64_t)IPW(ip, i*32+10) >> 12 | (uint64_t)IPX(ip, i*32+11) << 38;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*25+ 9) = (uint64_t)IPW(ip, i*32+11) >> 26 | (uint64_t)IPX(ip, i*32+12) << 24;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*25+10) = (uint64_t)IPW(ip, i*32+12) >> 40;\
- IP9(ip, i*32+13, parm); *((uint64_t *)op+i*25+10) |= (uint64_t)IPV(ip, i*32+13) << 10 | (uint64_t)IPX(ip, i*32+14) << 60;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*25+11) = (uint64_t)IPW(ip, i*32+14) >> 4 | (uint64_t)IPX(ip, i*32+15) << 46;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*25+12) = (uint64_t)IPW(ip, i*32+15) >> 18 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*25+13) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 18;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*25+14) = (uint64_t)IPW(ip, i*32+17) >> 46;\
- IP9(ip, i*32+18, parm); *((uint64_t *)op+i*25+14) |= (uint64_t)IPV(ip, i*32+18) << 4 | (uint64_t)IPX(ip, i*32+19) << 54;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*25+15) = (uint64_t)IPW(ip, i*32+19) >> 10 | (uint64_t)IPX(ip, i*32+20) << 40;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*25+16) = (uint64_t)IPW(ip, i*32+20) >> 24 | (uint64_t)IPX(ip, i*32+21) << 26;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*25+17) = (uint64_t)IPW(ip, i*32+21) >> 38;\
- IP9(ip, i*32+22, parm); *((uint64_t *)op+i*25+17) |= (uint64_t)IPV(ip, i*32+22) << 12 | (uint64_t)IPX(ip, i*32+23) << 62;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*25+18) = (uint64_t)IPW(ip, i*32+23) >> 2 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*25+19) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 34;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*25+20) = (uint64_t)IPW(ip, i*32+25) >> 30 | (uint64_t)IPX(ip, i*32+26) << 20;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*25+21) = (uint64_t)IPW(ip, i*32+26) >> 44;\
- IP9(ip, i*32+27, parm); *((uint64_t *)op+i*25+21) |= (uint64_t)IPV(ip, i*32+27) << 6 | (uint64_t)IPX(ip, i*32+28) << 56;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*25+22) = (uint64_t)IPW(ip, i*32+28) >> 8 | (uint64_t)IPX(ip, i*32+29) << 42;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*25+23) = (uint64_t)IPW(ip, i*32+29) >> 22 | (uint64_t)IPX(ip, i*32+30) << 28;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*25+24) = (uint64_t)IPW(ip, i*32+30) >> 36;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*25+24) |= (uint64_t)IPV(ip, i*32+31) << 14;\
-}
-
-#define BITPACK64_50(ip, op, parm) { \
- BITBLK64_50(ip, 0, op, parm); IPI(ip); op += 50*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_51(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*51+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 51;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*51+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 13 | (uint64_t)IPX(ip, i*64+2) << 38;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*51+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 26 | (uint64_t)IPX(ip, i*64+3) << 25;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*51+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 39;\
- IP9(ip, i*64+ 4, parm); *((uint64_t *)op+i*51+ 3) |= (uint64_t)IPV(ip, i*64+ 4) << 12 | (uint64_t)IPX(ip, i*64+5) << 63;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*51+ 4) = (uint64_t)IPW(ip, i*64+ 5) >> 1 | (uint64_t)IPX(ip, i*64+6) << 50;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*51+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 14 | (uint64_t)IPX(ip, i*64+7) << 37;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*51+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 27 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*51+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 40;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*51+ 7) |= (uint64_t)IPV(ip, i*64+ 9) << 11 | (uint64_t)IPX(ip, i*64+10) << 62;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*51+ 8) = (uint64_t)IPW(ip, i*64+10) >> 2 | (uint64_t)IPX(ip, i*64+11) << 49;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*51+ 9) = (uint64_t)IPW(ip, i*64+11) >> 15 | (uint64_t)IPX(ip, i*64+12) << 36;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*51+10) = (uint64_t)IPW(ip, i*64+12) >> 28 | (uint64_t)IPX(ip, i*64+13) << 23;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*51+11) = (uint64_t)IPW(ip, i*64+13) >> 41;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*51+11) |= (uint64_t)IPV(ip, i*64+14) << 10 | (uint64_t)IPX(ip, i*64+15) << 61;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*51+12) = (uint64_t)IPW(ip, i*64+15) >> 3 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*51+13) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 35;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*51+14) = (uint64_t)IPW(ip, i*64+17) >> 29 | (uint64_t)IPX(ip, i*64+18) << 22;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*51+15) = (uint64_t)IPW(ip, i*64+18) >> 42;\
- IP9(ip, i*64+19, parm); *((uint64_t *)op+i*51+15) |= (uint64_t)IPV(ip, i*64+19) << 9 | (uint64_t)IPX(ip, i*64+20) << 60;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*51+16) = (uint64_t)IPW(ip, i*64+20) >> 4 | (uint64_t)IPX(ip, i*64+21) << 47;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*51+17) = (uint64_t)IPW(ip, i*64+21) >> 17 | (uint64_t)IPX(ip, i*64+22) << 34;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*51+18) = (uint64_t)IPW(ip, i*64+22) >> 30 | (uint64_t)IPX(ip, i*64+23) << 21;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*51+19) = (uint64_t)IPW(ip, i*64+23) >> 43;\
- IP9(ip, i*64+24, parm); *((uint64_t *)op+i*51+19) |= (uint64_t)IPV(ip, i*64+24) << 8 | (uint64_t)IPX(ip, i*64+25) << 59;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*51+20) = (uint64_t)IPW(ip, i*64+25) >> 5 | (uint64_t)IPX(ip, i*64+26) << 46;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*51+21) = (uint64_t)IPW(ip, i*64+26) >> 18 | (uint64_t)IPX(ip, i*64+27) << 33;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*51+22) = (uint64_t)IPW(ip, i*64+27) >> 31 | (uint64_t)IPX(ip, i*64+28) << 20;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*51+23) = (uint64_t)IPW(ip, i*64+28) >> 44;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*51+23) |= (uint64_t)IPV(ip, i*64+29) << 7 | (uint64_t)IPX(ip, i*64+30) << 58;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*51+24) = (uint64_t)IPW(ip, i*64+30) >> 6 | (uint64_t)IPX(ip, i*64+31) << 45;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*51+25) = (uint64_t)IPW(ip, i*64+31) >> 19;\
-}
-
-#define BITPACK64_51(ip, op, parm) { \
- BITBLK64_51(ip, 0, op, parm); IPI(ip); op += 51*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_52(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*13+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 52;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*13+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 12 | (uint64_t)IPX(ip, i*16+2) << 40;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*13+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 24 | (uint64_t)IPX(ip, i*16+3) << 28;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*13+ 3) = (uint64_t)IPW(ip, i*16+ 3) >> 36 | (uint64_t)IPX(ip, i*16+4) << 16;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*13+ 4) = (uint64_t)IPW(ip, i*16+ 4) >> 48;\
- IP9(ip, i*16+ 5, parm); *((uint64_t *)op+i*13+ 4) |= (uint64_t)IPV(ip, i*16+ 5) << 4 | (uint64_t)IPX(ip, i*16+6) << 56;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*13+ 5) = (uint64_t)IPW(ip, i*16+ 6) >> 8 | (uint64_t)IPX(ip, i*16+7) << 44;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*13+ 6) = (uint64_t)IPW(ip, i*16+ 7) >> 20 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*13+ 7) = (uint64_t)IPW(ip, i*16+ 8) >> 32 | (uint64_t)IPX(ip, i*16+9) << 20;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*13+ 8) = (uint64_t)IPW(ip, i*16+ 9) >> 44;\
- IP9(ip, i*16+10, parm); *((uint64_t *)op+i*13+ 8) |= (uint64_t)IPV(ip, i*16+10) << 8 | (uint64_t)IPX(ip, i*16+11) << 60;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*13+ 9) = (uint64_t)IPW(ip, i*16+11) >> 4 | (uint64_t)IPX(ip, i*16+12) << 48;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*13+10) = (uint64_t)IPW(ip, i*16+12) >> 16 | (uint64_t)IPX(ip, i*16+13) << 36;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*13+11) = (uint64_t)IPW(ip, i*16+13) >> 28 | (uint64_t)IPX(ip, i*16+14) << 24;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*13+12) = (uint64_t)IPW(ip, i*16+14) >> 40;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*13+12) |= (uint64_t)IPV(ip, i*16+15) << 12;\
-}
-
-#define BITPACK64_52(ip, op, parm) { \
- BITBLK64_52(ip, 0, op, parm);\
- BITBLK64_52(ip, 1, op, parm); IPI(ip); op += 52*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_53(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*53+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 53;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*53+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 11 | (uint64_t)IPX(ip, i*64+2) << 42;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*53+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 22 | (uint64_t)IPX(ip, i*64+3) << 31;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*53+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 33 | (uint64_t)IPX(ip, i*64+4) << 20;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*53+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 44;\
- IP9(ip, i*64+ 5, parm); *((uint64_t *)op+i*53+ 4) |= (uint64_t)IPV(ip, i*64+ 5) << 9 | (uint64_t)IPX(ip, i*64+6) << 62;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*53+ 5) = (uint64_t)IPW(ip, i*64+ 6) >> 2 | (uint64_t)IPX(ip, i*64+7) << 51;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*53+ 6) = (uint64_t)IPW(ip, i*64+ 7) >> 13 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*53+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 29;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*53+ 8) = (uint64_t)IPW(ip, i*64+ 9) >> 35 | (uint64_t)IPX(ip, i*64+10) << 18;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*53+ 9) = (uint64_t)IPW(ip, i*64+10) >> 46;\
- IP9(ip, i*64+11, parm); *((uint64_t *)op+i*53+ 9) |= (uint64_t)IPV(ip, i*64+11) << 7 | (uint64_t)IPX(ip, i*64+12) << 60;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*53+10) = (uint64_t)IPW(ip, i*64+12) >> 4 | (uint64_t)IPX(ip, i*64+13) << 49;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*53+11) = (uint64_t)IPW(ip, i*64+13) >> 15 | (uint64_t)IPX(ip, i*64+14) << 38;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*53+12) = (uint64_t)IPW(ip, i*64+14) >> 26 | (uint64_t)IPX(ip, i*64+15) << 27;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*53+13) = (uint64_t)IPW(ip, i*64+15) >> 37 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*53+14) = (uint64_t)IPW(ip, i*64+16) >> 48;\
- IP9(ip, i*64+17, parm); *((uint64_t *)op+i*53+14) |= (uint64_t)IPV(ip, i*64+17) << 5 | (uint64_t)IPX(ip, i*64+18) << 58;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*53+15) = (uint64_t)IPW(ip, i*64+18) >> 6 | (uint64_t)IPX(ip, i*64+19) << 47;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*53+16) = (uint64_t)IPW(ip, i*64+19) >> 17 | (uint64_t)IPX(ip, i*64+20) << 36;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*53+17) = (uint64_t)IPW(ip, i*64+20) >> 28 | (uint64_t)IPX(ip, i*64+21) << 25;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*53+18) = (uint64_t)IPW(ip, i*64+21) >> 39 | (uint64_t)IPX(ip, i*64+22) << 14;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*53+19) = (uint64_t)IPW(ip, i*64+22) >> 50;\
- IP9(ip, i*64+23, parm); *((uint64_t *)op+i*53+19) |= (uint64_t)IPV(ip, i*64+23) << 3 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*53+20) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 45;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*53+21) = (uint64_t)IPW(ip, i*64+25) >> 19 | (uint64_t)IPX(ip, i*64+26) << 34;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*53+22) = (uint64_t)IPW(ip, i*64+26) >> 30 | (uint64_t)IPX(ip, i*64+27) << 23;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*53+23) = (uint64_t)IPW(ip, i*64+27) >> 41 | (uint64_t)IPX(ip, i*64+28) << 12;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*53+24) = (uint64_t)IPW(ip, i*64+28) >> 52;\
- IP9(ip, i*64+29, parm); *((uint64_t *)op+i*53+24) |= (uint64_t)IPV(ip, i*64+29) << 1 | (uint64_t)IPX(ip, i*64+30) << 54;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*53+25) = (uint64_t)IPW(ip, i*64+30) >> 10 | (uint64_t)IPX(ip, i*64+31) << 43;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*53+26) = (uint64_t)IPW(ip, i*64+31) >> 21;\
-}
-
-#define BITPACK64_53(ip, op, parm) { \
- BITBLK64_53(ip, 0, op, parm); IPI(ip); op += 53*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_54(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*27+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 54;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*27+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 10 | (uint64_t)IPX(ip, i*32+2) << 44;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*27+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 20 | (uint64_t)IPX(ip, i*32+3) << 34;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*27+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 30 | (uint64_t)IPX(ip, i*32+4) << 24;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*27+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 40 | (uint64_t)IPX(ip, i*32+5) << 14;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*27+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 50;\
- IP9(ip, i*32+ 6, parm); *((uint64_t *)op+i*27+ 5) |= (uint64_t)IPV(ip, i*32+ 6) << 4 | (uint64_t)IPX(ip, i*32+7) << 58;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*27+ 6) = (uint64_t)IPW(ip, i*32+ 7) >> 6 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*27+ 7) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 38;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*27+ 8) = (uint64_t)IPW(ip, i*32+ 9) >> 26 | (uint64_t)IPX(ip, i*32+10) << 28;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*27+ 9) = (uint64_t)IPW(ip, i*32+10) >> 36 | (uint64_t)IPX(ip, i*32+11) << 18;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*27+10) = (uint64_t)IPW(ip, i*32+11) >> 46;\
- IP9(ip, i*32+12, parm); *((uint64_t *)op+i*27+10) |= (uint64_t)IPV(ip, i*32+12) << 8 | (uint64_t)IPX(ip, i*32+13) << 62;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*27+11) = (uint64_t)IPW(ip, i*32+13) >> 2 | (uint64_t)IPX(ip, i*32+14) << 52;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*27+12) = (uint64_t)IPW(ip, i*32+14) >> 12 | (uint64_t)IPX(ip, i*32+15) << 42;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*27+13) = (uint64_t)IPW(ip, i*32+15) >> 22 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*27+14) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 22;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*27+15) = (uint64_t)IPW(ip, i*32+17) >> 42 | (uint64_t)IPX(ip, i*32+18) << 12;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*27+16) = (uint64_t)IPW(ip, i*32+18) >> 52;\
- IP9(ip, i*32+19, parm); *((uint64_t *)op+i*27+16) |= (uint64_t)IPV(ip, i*32+19) << 2 | (uint64_t)IPX(ip, i*32+20) << 56;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*27+17) = (uint64_t)IPW(ip, i*32+20) >> 8 | (uint64_t)IPX(ip, i*32+21) << 46;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*27+18) = (uint64_t)IPW(ip, i*32+21) >> 18 | (uint64_t)IPX(ip, i*32+22) << 36;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*27+19) = (uint64_t)IPW(ip, i*32+22) >> 28 | (uint64_t)IPX(ip, i*32+23) << 26;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*27+20) = (uint64_t)IPW(ip, i*32+23) >> 38 | (uint64_t)IPX(ip, i*32+24) << 16;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*27+21) = (uint64_t)IPW(ip, i*32+24) >> 48;\
- IP9(ip, i*32+25, parm); *((uint64_t *)op+i*27+21) |= (uint64_t)IPV(ip, i*32+25) << 6 | (uint64_t)IPX(ip, i*32+26) << 60;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*27+22) = (uint64_t)IPW(ip, i*32+26) >> 4 | (uint64_t)IPX(ip, i*32+27) << 50;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*27+23) = (uint64_t)IPW(ip, i*32+27) >> 14 | (uint64_t)IPX(ip, i*32+28) << 40;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*27+24) = (uint64_t)IPW(ip, i*32+28) >> 24 | (uint64_t)IPX(ip, i*32+29) << 30;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*27+25) = (uint64_t)IPW(ip, i*32+29) >> 34 | (uint64_t)IPX(ip, i*32+30) << 20;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*27+26) = (uint64_t)IPW(ip, i*32+30) >> 44;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*27+26) |= (uint64_t)IPV(ip, i*32+31) << 10;\
-}
-
-#define BITPACK64_54(ip, op, parm) { \
- BITBLK64_54(ip, 0, op, parm); IPI(ip); op += 54*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_55(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*55+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 55;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*55+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 9 | (uint64_t)IPX(ip, i*64+2) << 46;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*55+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 18 | (uint64_t)IPX(ip, i*64+3) << 37;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*55+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 27 | (uint64_t)IPX(ip, i*64+4) << 28;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*55+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 36 | (uint64_t)IPX(ip, i*64+5) << 19;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*55+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 45 | (uint64_t)IPX(ip, i*64+6) << 10;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*55+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 54;\
- IP9(ip, i*64+ 7, parm); *((uint64_t *)op+i*55+ 6) |= (uint64_t)IPV(ip, i*64+ 7) << 1 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*55+ 7) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 47;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*55+ 8) = (uint64_t)IPW(ip, i*64+ 9) >> 17 | (uint64_t)IPX(ip, i*64+10) << 38;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*55+ 9) = (uint64_t)IPW(ip, i*64+10) >> 26 | (uint64_t)IPX(ip, i*64+11) << 29;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*55+10) = (uint64_t)IPW(ip, i*64+11) >> 35 | (uint64_t)IPX(ip, i*64+12) << 20;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*55+11) = (uint64_t)IPW(ip, i*64+12) >> 44 | (uint64_t)IPX(ip, i*64+13) << 11;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*55+12) = (uint64_t)IPW(ip, i*64+13) >> 53;\
- IP9(ip, i*64+14, parm); *((uint64_t *)op+i*55+12) |= (uint64_t)IPV(ip, i*64+14) << 2 | (uint64_t)IPX(ip, i*64+15) << 57;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*55+13) = (uint64_t)IPW(ip, i*64+15) >> 7 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*55+14) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 39;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*55+15) = (uint64_t)IPW(ip, i*64+17) >> 25 | (uint64_t)IPX(ip, i*64+18) << 30;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*55+16) = (uint64_t)IPW(ip, i*64+18) >> 34 | (uint64_t)IPX(ip, i*64+19) << 21;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*55+17) = (uint64_t)IPW(ip, i*64+19) >> 43 | (uint64_t)IPX(ip, i*64+20) << 12;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*55+18) = (uint64_t)IPW(ip, i*64+20) >> 52;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*55+18) |= (uint64_t)IPV(ip, i*64+21) << 3 | (uint64_t)IPX(ip, i*64+22) << 58;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*55+19) = (uint64_t)IPW(ip, i*64+22) >> 6 | (uint64_t)IPX(ip, i*64+23) << 49;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*55+20) = (uint64_t)IPW(ip, i*64+23) >> 15 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*55+21) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 31;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*55+22) = (uint64_t)IPW(ip, i*64+25) >> 33 | (uint64_t)IPX(ip, i*64+26) << 22;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*55+23) = (uint64_t)IPW(ip, i*64+26) >> 42 | (uint64_t)IPX(ip, i*64+27) << 13;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*55+24) = (uint64_t)IPW(ip, i*64+27) >> 51;\
- IP9(ip, i*64+28, parm); *((uint64_t *)op+i*55+24) |= (uint64_t)IPV(ip, i*64+28) << 4 | (uint64_t)IPX(ip, i*64+29) << 59;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*55+25) = (uint64_t)IPW(ip, i*64+29) >> 5 | (uint64_t)IPX(ip, i*64+30) << 50;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*55+26) = (uint64_t)IPW(ip, i*64+30) >> 14 | (uint64_t)IPX(ip, i*64+31) << 41;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*55+27) = (uint64_t)IPW(ip, i*64+31) >> 23;\
-}
-
-#define BITPACK64_55(ip, op, parm) { \
- BITBLK64_55(ip, 0, op, parm); IPI(ip); op += 55*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_56(ip, i, op, parm) { ;\
- IP9(ip, i*8+ 0, parm); *((uint64_t *)op+i*7+ 0) = (uint64_t)IPV(ip, i*8+ 0) | (uint64_t)IPX(ip, i*8+1) << 56;\
- IP64(ip, i*8+ 1, parm); *((uint64_t *)op+i*7+ 1) = (uint64_t)IPW(ip, i*8+ 1) >> 8 | (uint64_t)IPX(ip, i*8+2) << 48;\
- IP64(ip, i*8+ 2, parm); *((uint64_t *)op+i*7+ 2) = (uint64_t)IPW(ip, i*8+ 2) >> 16 | (uint64_t)IPX(ip, i*8+3) << 40;\
- IP64(ip, i*8+ 3, parm); *((uint64_t *)op+i*7+ 3) = (uint64_t)IPW(ip, i*8+ 3) >> 24 | (uint64_t)IPX(ip, i*8+4) << 32;\
- IP64(ip, i*8+ 4, parm); *((uint64_t *)op+i*7+ 4) = (uint64_t)IPW(ip, i*8+ 4) >> 32 | (uint64_t)IPX(ip, i*8+5) << 24;\
- IP64(ip, i*8+ 5, parm); *((uint64_t *)op+i*7+ 5) = (uint64_t)IPW(ip, i*8+ 5) >> 40 | (uint64_t)IPX(ip, i*8+6) << 16;\
- IP64(ip, i*8+ 6, parm); *((uint64_t *)op+i*7+ 6) = (uint64_t)IPW(ip, i*8+ 6) >> 48;\
- IP9(ip, i*8+ 7, parm); *((uint64_t *)op+i*7+ 6) |= (uint64_t)IPV(ip, i*8+ 7) << 8;\
-}
-
-#define BITPACK64_56(ip, op, parm) { \
- BITBLK64_56(ip, 0, op, parm);\
- BITBLK64_56(ip, 1, op, parm);\
- BITBLK64_56(ip, 2, op, parm);\
- BITBLK64_56(ip, 3, op, parm); IPI(ip); op += 56*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_57(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*57+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 57;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*57+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 7 | (uint64_t)IPX(ip, i*64+2) << 50;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*57+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 14 | (uint64_t)IPX(ip, i*64+3) << 43;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*57+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 21 | (uint64_t)IPX(ip, i*64+4) << 36;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*57+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 28 | (uint64_t)IPX(ip, i*64+5) << 29;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*57+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 35 | (uint64_t)IPX(ip, i*64+6) << 22;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*57+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 42 | (uint64_t)IPX(ip, i*64+7) << 15;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*57+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 49 | (uint64_t)IPX(ip, i*64+8) << 8;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*57+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 56;\
- IP9(ip, i*64+ 9, parm); *((uint64_t *)op+i*57+ 8) |= (uint64_t)IPV(ip, i*64+ 9) << 1 | (uint64_t)IPX(ip, i*64+10) << 58;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*57+ 9) = (uint64_t)IPW(ip, i*64+10) >> 6 | (uint64_t)IPX(ip, i*64+11) << 51;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*57+10) = (uint64_t)IPW(ip, i*64+11) >> 13 | (uint64_t)IPX(ip, i*64+12) << 44;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*57+11) = (uint64_t)IPW(ip, i*64+12) >> 20 | (uint64_t)IPX(ip, i*64+13) << 37;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*57+12) = (uint64_t)IPW(ip, i*64+13) >> 27 | (uint64_t)IPX(ip, i*64+14) << 30;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*57+13) = (uint64_t)IPW(ip, i*64+14) >> 34 | (uint64_t)IPX(ip, i*64+15) << 23;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*57+14) = (uint64_t)IPW(ip, i*64+15) >> 41 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*57+15) = (uint64_t)IPW(ip, i*64+16) >> 48 | (uint64_t)IPX(ip, i*64+17) << 9;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*57+16) = (uint64_t)IPW(ip, i*64+17) >> 55;\
- IP9(ip, i*64+18, parm); *((uint64_t *)op+i*57+16) |= (uint64_t)IPV(ip, i*64+18) << 2 | (uint64_t)IPX(ip, i*64+19) << 59;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*57+17) = (uint64_t)IPW(ip, i*64+19) >> 5 | (uint64_t)IPX(ip, i*64+20) << 52;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*57+18) = (uint64_t)IPW(ip, i*64+20) >> 12 | (uint64_t)IPX(ip, i*64+21) << 45;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*57+19) = (uint64_t)IPW(ip, i*64+21) >> 19 | (uint64_t)IPX(ip, i*64+22) << 38;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*57+20) = (uint64_t)IPW(ip, i*64+22) >> 26 | (uint64_t)IPX(ip, i*64+23) << 31;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*57+21) = (uint64_t)IPW(ip, i*64+23) >> 33 | (uint64_t)IPX(ip, i*64+24) << 24;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*57+22) = (uint64_t)IPW(ip, i*64+24) >> 40 | (uint64_t)IPX(ip, i*64+25) << 17;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*57+23) = (uint64_t)IPW(ip, i*64+25) >> 47 | (uint64_t)IPX(ip, i*64+26) << 10;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*57+24) = (uint64_t)IPW(ip, i*64+26) >> 54;\
- IP9(ip, i*64+27, parm); *((uint64_t *)op+i*57+24) |= (uint64_t)IPV(ip, i*64+27) << 3 | (uint64_t)IPX(ip, i*64+28) << 60;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*57+25) = (uint64_t)IPW(ip, i*64+28) >> 4 | (uint64_t)IPX(ip, i*64+29) << 53;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*57+26) = (uint64_t)IPW(ip, i*64+29) >> 11 | (uint64_t)IPX(ip, i*64+30) << 46;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*57+27) = (uint64_t)IPW(ip, i*64+30) >> 18 | (uint64_t)IPX(ip, i*64+31) << 39;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*57+28) = (uint64_t)IPW(ip, i*64+31) >> 25;\
-}
-
-#define BITPACK64_57(ip, op, parm) { \
- BITBLK64_57(ip, 0, op, parm); IPI(ip); op += 57*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_58(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*29+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 58;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*29+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 6 | (uint64_t)IPX(ip, i*32+2) << 52;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*29+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 12 | (uint64_t)IPX(ip, i*32+3) << 46;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*29+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 18 | (uint64_t)IPX(ip, i*32+4) << 40;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*29+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 24 | (uint64_t)IPX(ip, i*32+5) << 34;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*29+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 30 | (uint64_t)IPX(ip, i*32+6) << 28;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*29+ 6) = (uint64_t)IPW(ip, i*32+ 6) >> 36 | (uint64_t)IPX(ip, i*32+7) << 22;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*29+ 7) = (uint64_t)IPW(ip, i*32+ 7) >> 42 | (uint64_t)IPX(ip, i*32+8) << 16;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*29+ 8) = (uint64_t)IPW(ip, i*32+ 8) >> 48 | (uint64_t)IPX(ip, i*32+9) << 10;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*29+ 9) = (uint64_t)IPW(ip, i*32+ 9) >> 54;\
- IP9(ip, i*32+10, parm); *((uint64_t *)op+i*29+ 9) |= (uint64_t)IPV(ip, i*32+10) << 4 | (uint64_t)IPX(ip, i*32+11) << 62;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*29+10) = (uint64_t)IPW(ip, i*32+11) >> 2 | (uint64_t)IPX(ip, i*32+12) << 56;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*29+11) = (uint64_t)IPW(ip, i*32+12) >> 8 | (uint64_t)IPX(ip, i*32+13) << 50;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*29+12) = (uint64_t)IPW(ip, i*32+13) >> 14 | (uint64_t)IPX(ip, i*32+14) << 44;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*29+13) = (uint64_t)IPW(ip, i*32+14) >> 20 | (uint64_t)IPX(ip, i*32+15) << 38;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*29+14) = (uint64_t)IPW(ip, i*32+15) >> 26 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*29+15) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 26;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*29+16) = (uint64_t)IPW(ip, i*32+17) >> 38 | (uint64_t)IPX(ip, i*32+18) << 20;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*29+17) = (uint64_t)IPW(ip, i*32+18) >> 44 | (uint64_t)IPX(ip, i*32+19) << 14;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*29+18) = (uint64_t)IPW(ip, i*32+19) >> 50 | (uint64_t)IPX(ip, i*32+20) << 8;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*29+19) = (uint64_t)IPW(ip, i*32+20) >> 56;\
- IP9(ip, i*32+21, parm); *((uint64_t *)op+i*29+19) |= (uint64_t)IPV(ip, i*32+21) << 2 | (uint64_t)IPX(ip, i*32+22) << 60;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*29+20) = (uint64_t)IPW(ip, i*32+22) >> 4 | (uint64_t)IPX(ip, i*32+23) << 54;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*29+21) = (uint64_t)IPW(ip, i*32+23) >> 10 | (uint64_t)IPX(ip, i*32+24) << 48;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*29+22) = (uint64_t)IPW(ip, i*32+24) >> 16 | (uint64_t)IPX(ip, i*32+25) << 42;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*29+23) = (uint64_t)IPW(ip, i*32+25) >> 22 | (uint64_t)IPX(ip, i*32+26) << 36;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*29+24) = (uint64_t)IPW(ip, i*32+26) >> 28 | (uint64_t)IPX(ip, i*32+27) << 30;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*29+25) = (uint64_t)IPW(ip, i*32+27) >> 34 | (uint64_t)IPX(ip, i*32+28) << 24;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*29+26) = (uint64_t)IPW(ip, i*32+28) >> 40 | (uint64_t)IPX(ip, i*32+29) << 18;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*29+27) = (uint64_t)IPW(ip, i*32+29) >> 46 | (uint64_t)IPX(ip, i*32+30) << 12;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*29+28) = (uint64_t)IPW(ip, i*32+30) >> 52;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*29+28) |= (uint64_t)IPV(ip, i*32+31) << 6;\
-}
-
-#define BITPACK64_58(ip, op, parm) { \
- BITBLK64_58(ip, 0, op, parm); IPI(ip); op += 58*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_59(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*59+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 59;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*59+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 5 | (uint64_t)IPX(ip, i*64+2) << 54;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*59+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 10 | (uint64_t)IPX(ip, i*64+3) << 49;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*59+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 15 | (uint64_t)IPX(ip, i*64+4) << 44;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*59+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 20 | (uint64_t)IPX(ip, i*64+5) << 39;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*59+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 25 | (uint64_t)IPX(ip, i*64+6) << 34;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*59+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 30 | (uint64_t)IPX(ip, i*64+7) << 29;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*59+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 35 | (uint64_t)IPX(ip, i*64+8) << 24;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*59+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 40 | (uint64_t)IPX(ip, i*64+9) << 19;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*59+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 45 | (uint64_t)IPX(ip, i*64+10) << 14;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*59+10) = (uint64_t)IPW(ip, i*64+10) >> 50 | (uint64_t)IPX(ip, i*64+11) << 9;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*59+11) = (uint64_t)IPW(ip, i*64+11) >> 55;\
- IP9(ip, i*64+12, parm); *((uint64_t *)op+i*59+11) |= (uint64_t)IPV(ip, i*64+12) << 4 | (uint64_t)IPX(ip, i*64+13) << 63;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*59+12) = (uint64_t)IPW(ip, i*64+13) >> 1 | (uint64_t)IPX(ip, i*64+14) << 58;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*59+13) = (uint64_t)IPW(ip, i*64+14) >> 6 | (uint64_t)IPX(ip, i*64+15) << 53;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*59+14) = (uint64_t)IPW(ip, i*64+15) >> 11 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*59+15) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 43;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*59+16) = (uint64_t)IPW(ip, i*64+17) >> 21 | (uint64_t)IPX(ip, i*64+18) << 38;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*59+17) = (uint64_t)IPW(ip, i*64+18) >> 26 | (uint64_t)IPX(ip, i*64+19) << 33;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*59+18) = (uint64_t)IPW(ip, i*64+19) >> 31 | (uint64_t)IPX(ip, i*64+20) << 28;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*59+19) = (uint64_t)IPW(ip, i*64+20) >> 36 | (uint64_t)IPX(ip, i*64+21) << 23;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*59+20) = (uint64_t)IPW(ip, i*64+21) >> 41 | (uint64_t)IPX(ip, i*64+22) << 18;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*59+21) = (uint64_t)IPW(ip, i*64+22) >> 46 | (uint64_t)IPX(ip, i*64+23) << 13;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*59+22) = (uint64_t)IPW(ip, i*64+23) >> 51 | (uint64_t)IPX(ip, i*64+24) << 8;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*59+23) = (uint64_t)IPW(ip, i*64+24) >> 56;\
- IP9(ip, i*64+25, parm); *((uint64_t *)op+i*59+23) |= (uint64_t)IPV(ip, i*64+25) << 3 | (uint64_t)IPX(ip, i*64+26) << 62;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*59+24) = (uint64_t)IPW(ip, i*64+26) >> 2 | (uint64_t)IPX(ip, i*64+27) << 57;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*59+25) = (uint64_t)IPW(ip, i*64+27) >> 7 | (uint64_t)IPX(ip, i*64+28) << 52;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*59+26) = (uint64_t)IPW(ip, i*64+28) >> 12 | (uint64_t)IPX(ip, i*64+29) << 47;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*59+27) = (uint64_t)IPW(ip, i*64+29) >> 17 | (uint64_t)IPX(ip, i*64+30) << 42;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*59+28) = (uint64_t)IPW(ip, i*64+30) >> 22 | (uint64_t)IPX(ip, i*64+31) << 37;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*59+29) = (uint64_t)IPW(ip, i*64+31) >> 27;\
-}
-
-#define BITPACK64_59(ip, op, parm) { \
- BITBLK64_59(ip, 0, op, parm); IPI(ip); op += 59*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_60(ip, i, op, parm) { ;\
- IP9(ip, i*16+ 0, parm); *((uint64_t *)op+i*15+ 0) = (uint64_t)IPV(ip, i*16+ 0) | (uint64_t)IPX(ip, i*16+1) << 60;\
- IP64(ip, i*16+ 1, parm); *((uint64_t *)op+i*15+ 1) = (uint64_t)IPW(ip, i*16+ 1) >> 4 | (uint64_t)IPX(ip, i*16+2) << 56;\
- IP64(ip, i*16+ 2, parm); *((uint64_t *)op+i*15+ 2) = (uint64_t)IPW(ip, i*16+ 2) >> 8 | (uint64_t)IPX(ip, i*16+3) << 52;\
- IP64(ip, i*16+ 3, parm); *((uint64_t *)op+i*15+ 3) = (uint64_t)IPW(ip, i*16+ 3) >> 12 | (uint64_t)IPX(ip, i*16+4) << 48;\
- IP64(ip, i*16+ 4, parm); *((uint64_t *)op+i*15+ 4) = (uint64_t)IPW(ip, i*16+ 4) >> 16 | (uint64_t)IPX(ip, i*16+5) << 44;\
- IP64(ip, i*16+ 5, parm); *((uint64_t *)op+i*15+ 5) = (uint64_t)IPW(ip, i*16+ 5) >> 20 | (uint64_t)IPX(ip, i*16+6) << 40;\
- IP64(ip, i*16+ 6, parm); *((uint64_t *)op+i*15+ 6) = (uint64_t)IPW(ip, i*16+ 6) >> 24 | (uint64_t)IPX(ip, i*16+7) << 36;\
- IP64(ip, i*16+ 7, parm); *((uint64_t *)op+i*15+ 7) = (uint64_t)IPW(ip, i*16+ 7) >> 28 | (uint64_t)IPX(ip, i*16+8) << 32;\
- IP64(ip, i*16+ 8, parm); *((uint64_t *)op+i*15+ 8) = (uint64_t)IPW(ip, i*16+ 8) >> 32 | (uint64_t)IPX(ip, i*16+9) << 28;\
- IP64(ip, i*16+ 9, parm); *((uint64_t *)op+i*15+ 9) = (uint64_t)IPW(ip, i*16+ 9) >> 36 | (uint64_t)IPX(ip, i*16+10) << 24;\
- IP64(ip, i*16+10, parm); *((uint64_t *)op+i*15+10) = (uint64_t)IPW(ip, i*16+10) >> 40 | (uint64_t)IPX(ip, i*16+11) << 20;\
- IP64(ip, i*16+11, parm); *((uint64_t *)op+i*15+11) = (uint64_t)IPW(ip, i*16+11) >> 44 | (uint64_t)IPX(ip, i*16+12) << 16;\
- IP64(ip, i*16+12, parm); *((uint64_t *)op+i*15+12) = (uint64_t)IPW(ip, i*16+12) >> 48 | (uint64_t)IPX(ip, i*16+13) << 12;\
- IP64(ip, i*16+13, parm); *((uint64_t *)op+i*15+13) = (uint64_t)IPW(ip, i*16+13) >> 52 | (uint64_t)IPX(ip, i*16+14) << 8;\
- IP64(ip, i*16+14, parm); *((uint64_t *)op+i*15+14) = (uint64_t)IPW(ip, i*16+14) >> 56;\
- IP9(ip, i*16+15, parm); *((uint64_t *)op+i*15+14) |= (uint64_t)IPV(ip, i*16+15) << 4;\
-}
-
-#define BITPACK64_60(ip, op, parm) { \
- BITBLK64_60(ip, 0, op, parm);\
- BITBLK64_60(ip, 1, op, parm); IPI(ip); op += 60*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_61(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*61+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 61;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*61+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 3 | (uint64_t)IPX(ip, i*64+2) << 58;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*61+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 6 | (uint64_t)IPX(ip, i*64+3) << 55;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*61+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 9 | (uint64_t)IPX(ip, i*64+4) << 52;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*61+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 12 | (uint64_t)IPX(ip, i*64+5) << 49;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*61+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 15 | (uint64_t)IPX(ip, i*64+6) << 46;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*61+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 18 | (uint64_t)IPX(ip, i*64+7) << 43;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*61+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 21 | (uint64_t)IPX(ip, i*64+8) << 40;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*61+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 24 | (uint64_t)IPX(ip, i*64+9) << 37;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*61+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 27 | (uint64_t)IPX(ip, i*64+10) << 34;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*61+10) = (uint64_t)IPW(ip, i*64+10) >> 30 | (uint64_t)IPX(ip, i*64+11) << 31;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*61+11) = (uint64_t)IPW(ip, i*64+11) >> 33 | (uint64_t)IPX(ip, i*64+12) << 28;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*61+12) = (uint64_t)IPW(ip, i*64+12) >> 36 | (uint64_t)IPX(ip, i*64+13) << 25;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*61+13) = (uint64_t)IPW(ip, i*64+13) >> 39 | (uint64_t)IPX(ip, i*64+14) << 22;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*61+14) = (uint64_t)IPW(ip, i*64+14) >> 42 | (uint64_t)IPX(ip, i*64+15) << 19;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*61+15) = (uint64_t)IPW(ip, i*64+15) >> 45 | (uint64_t)IPX(ip, i*64+16) << 16;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*61+16) = (uint64_t)IPW(ip, i*64+16) >> 48 | (uint64_t)IPX(ip, i*64+17) << 13;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*61+17) = (uint64_t)IPW(ip, i*64+17) >> 51 | (uint64_t)IPX(ip, i*64+18) << 10;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*61+18) = (uint64_t)IPW(ip, i*64+18) >> 54 | (uint64_t)IPX(ip, i*64+19) << 7;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*61+19) = (uint64_t)IPW(ip, i*64+19) >> 57 | (uint64_t)IPX(ip, i*64+20) << 4;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*61+20) = (uint64_t)IPW(ip, i*64+20) >> 60;\
- IP9(ip, i*64+21, parm); *((uint64_t *)op+i*61+20) |= (uint64_t)IPV(ip, i*64+21) << 1 | (uint64_t)IPX(ip, i*64+22) << 62;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*61+21) = (uint64_t)IPW(ip, i*64+22) >> 2 | (uint64_t)IPX(ip, i*64+23) << 59;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*61+22) = (uint64_t)IPW(ip, i*64+23) >> 5 | (uint64_t)IPX(ip, i*64+24) << 56;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*61+23) = (uint64_t)IPW(ip, i*64+24) >> 8 | (uint64_t)IPX(ip, i*64+25) << 53;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*61+24) = (uint64_t)IPW(ip, i*64+25) >> 11 | (uint64_t)IPX(ip, i*64+26) << 50;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*61+25) = (uint64_t)IPW(ip, i*64+26) >> 14 | (uint64_t)IPX(ip, i*64+27) << 47;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*61+26) = (uint64_t)IPW(ip, i*64+27) >> 17 | (uint64_t)IPX(ip, i*64+28) << 44;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*61+27) = (uint64_t)IPW(ip, i*64+28) >> 20 | (uint64_t)IPX(ip, i*64+29) << 41;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*61+28) = (uint64_t)IPW(ip, i*64+29) >> 23 | (uint64_t)IPX(ip, i*64+30) << 38;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*61+29) = (uint64_t)IPW(ip, i*64+30) >> 26 | (uint64_t)IPX(ip, i*64+31) << 35;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*61+30) = (uint64_t)IPW(ip, i*64+31) >> 29;\
-}
-
-#define BITPACK64_61(ip, op, parm) { \
- BITBLK64_61(ip, 0, op, parm); IPI(ip); op += 61*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_62(ip, i, op, parm) { ;\
- IP9(ip, i*32+ 0, parm); *((uint64_t *)op+i*31+ 0) = (uint64_t)IPV(ip, i*32+ 0) | (uint64_t)IPX(ip, i*32+1) << 62;\
- IP64(ip, i*32+ 1, parm); *((uint64_t *)op+i*31+ 1) = (uint64_t)IPW(ip, i*32+ 1) >> 2 | (uint64_t)IPX(ip, i*32+2) << 60;\
- IP64(ip, i*32+ 2, parm); *((uint64_t *)op+i*31+ 2) = (uint64_t)IPW(ip, i*32+ 2) >> 4 | (uint64_t)IPX(ip, i*32+3) << 58;\
- IP64(ip, i*32+ 3, parm); *((uint64_t *)op+i*31+ 3) = (uint64_t)IPW(ip, i*32+ 3) >> 6 | (uint64_t)IPX(ip, i*32+4) << 56;\
- IP64(ip, i*32+ 4, parm); *((uint64_t *)op+i*31+ 4) = (uint64_t)IPW(ip, i*32+ 4) >> 8 | (uint64_t)IPX(ip, i*32+5) << 54;\
- IP64(ip, i*32+ 5, parm); *((uint64_t *)op+i*31+ 5) = (uint64_t)IPW(ip, i*32+ 5) >> 10 | (uint64_t)IPX(ip, i*32+6) << 52;\
- IP64(ip, i*32+ 6, parm); *((uint64_t *)op+i*31+ 6) = (uint64_t)IPW(ip, i*32+ 6) >> 12 | (uint64_t)IPX(ip, i*32+7) << 50;\
- IP64(ip, i*32+ 7, parm); *((uint64_t *)op+i*31+ 7) = (uint64_t)IPW(ip, i*32+ 7) >> 14 | (uint64_t)IPX(ip, i*32+8) << 48;\
- IP64(ip, i*32+ 8, parm); *((uint64_t *)op+i*31+ 8) = (uint64_t)IPW(ip, i*32+ 8) >> 16 | (uint64_t)IPX(ip, i*32+9) << 46;\
- IP64(ip, i*32+ 9, parm); *((uint64_t *)op+i*31+ 9) = (uint64_t)IPW(ip, i*32+ 9) >> 18 | (uint64_t)IPX(ip, i*32+10) << 44;\
- IP64(ip, i*32+10, parm); *((uint64_t *)op+i*31+10) = (uint64_t)IPW(ip, i*32+10) >> 20 | (uint64_t)IPX(ip, i*32+11) << 42;\
- IP64(ip, i*32+11, parm); *((uint64_t *)op+i*31+11) = (uint64_t)IPW(ip, i*32+11) >> 22 | (uint64_t)IPX(ip, i*32+12) << 40;\
- IP64(ip, i*32+12, parm); *((uint64_t *)op+i*31+12) = (uint64_t)IPW(ip, i*32+12) >> 24 | (uint64_t)IPX(ip, i*32+13) << 38;\
- IP64(ip, i*32+13, parm); *((uint64_t *)op+i*31+13) = (uint64_t)IPW(ip, i*32+13) >> 26 | (uint64_t)IPX(ip, i*32+14) << 36;\
- IP64(ip, i*32+14, parm); *((uint64_t *)op+i*31+14) = (uint64_t)IPW(ip, i*32+14) >> 28 | (uint64_t)IPX(ip, i*32+15) << 34;\
- IP64(ip, i*32+15, parm); *((uint64_t *)op+i*31+15) = (uint64_t)IPW(ip, i*32+15) >> 30 | (uint64_t)IPX(ip, i*32+16) << 32;\
- IP64(ip, i*32+16, parm); *((uint64_t *)op+i*31+16) = (uint64_t)IPW(ip, i*32+16) >> 32 | (uint64_t)IPX(ip, i*32+17) << 30;\
- IP64(ip, i*32+17, parm); *((uint64_t *)op+i*31+17) = (uint64_t)IPW(ip, i*32+17) >> 34 | (uint64_t)IPX(ip, i*32+18) << 28;\
- IP64(ip, i*32+18, parm); *((uint64_t *)op+i*31+18) = (uint64_t)IPW(ip, i*32+18) >> 36 | (uint64_t)IPX(ip, i*32+19) << 26;\
- IP64(ip, i*32+19, parm); *((uint64_t *)op+i*31+19) = (uint64_t)IPW(ip, i*32+19) >> 38 | (uint64_t)IPX(ip, i*32+20) << 24;\
- IP64(ip, i*32+20, parm); *((uint64_t *)op+i*31+20) = (uint64_t)IPW(ip, i*32+20) >> 40 | (uint64_t)IPX(ip, i*32+21) << 22;\
- IP64(ip, i*32+21, parm); *((uint64_t *)op+i*31+21) = (uint64_t)IPW(ip, i*32+21) >> 42 | (uint64_t)IPX(ip, i*32+22) << 20;\
- IP64(ip, i*32+22, parm); *((uint64_t *)op+i*31+22) = (uint64_t)IPW(ip, i*32+22) >> 44 | (uint64_t)IPX(ip, i*32+23) << 18;\
- IP64(ip, i*32+23, parm); *((uint64_t *)op+i*31+23) = (uint64_t)IPW(ip, i*32+23) >> 46 | (uint64_t)IPX(ip, i*32+24) << 16;\
- IP64(ip, i*32+24, parm); *((uint64_t *)op+i*31+24) = (uint64_t)IPW(ip, i*32+24) >> 48 | (uint64_t)IPX(ip, i*32+25) << 14;\
- IP64(ip, i*32+25, parm); *((uint64_t *)op+i*31+25) = (uint64_t)IPW(ip, i*32+25) >> 50 | (uint64_t)IPX(ip, i*32+26) << 12;\
- IP64(ip, i*32+26, parm); *((uint64_t *)op+i*31+26) = (uint64_t)IPW(ip, i*32+26) >> 52 | (uint64_t)IPX(ip, i*32+27) << 10;\
- IP64(ip, i*32+27, parm); *((uint64_t *)op+i*31+27) = (uint64_t)IPW(ip, i*32+27) >> 54 | (uint64_t)IPX(ip, i*32+28) << 8;\
- IP64(ip, i*32+28, parm); *((uint64_t *)op+i*31+28) = (uint64_t)IPW(ip, i*32+28) >> 56 | (uint64_t)IPX(ip, i*32+29) << 6;\
- IP64(ip, i*32+29, parm); *((uint64_t *)op+i*31+29) = (uint64_t)IPW(ip, i*32+29) >> 58 | (uint64_t)IPX(ip, i*32+30) << 4;\
- IP64(ip, i*32+30, parm); *((uint64_t *)op+i*31+30) = (uint64_t)IPW(ip, i*32+30) >> 60;\
- IP9(ip, i*32+31, parm); *((uint64_t *)op+i*31+30) |= (uint64_t)IPV(ip, i*32+31) << 2;\
-}
-
-#define BITPACK64_62(ip, op, parm) { \
- BITBLK64_62(ip, 0, op, parm); IPI(ip); op += 62*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_63(ip, i, op, parm) { ;\
- IP9(ip, i*64+ 0, parm); *((uint64_t *)op+i*63+ 0) = (uint64_t)IPV(ip, i*64+ 0) | (uint64_t)IPX(ip, i*64+1) << 63;\
- IP64(ip, i*64+ 1, parm); *((uint64_t *)op+i*63+ 1) = (uint64_t)IPW(ip, i*64+ 1) >> 1 | (uint64_t)IPX(ip, i*64+2) << 62;\
- IP64(ip, i*64+ 2, parm); *((uint64_t *)op+i*63+ 2) = (uint64_t)IPW(ip, i*64+ 2) >> 2 | (uint64_t)IPX(ip, i*64+3) << 61;\
- IP64(ip, i*64+ 3, parm); *((uint64_t *)op+i*63+ 3) = (uint64_t)IPW(ip, i*64+ 3) >> 3 | (uint64_t)IPX(ip, i*64+4) << 60;\
- IP64(ip, i*64+ 4, parm); *((uint64_t *)op+i*63+ 4) = (uint64_t)IPW(ip, i*64+ 4) >> 4 | (uint64_t)IPX(ip, i*64+5) << 59;\
- IP64(ip, i*64+ 5, parm); *((uint64_t *)op+i*63+ 5) = (uint64_t)IPW(ip, i*64+ 5) >> 5 | (uint64_t)IPX(ip, i*64+6) << 58;\
- IP64(ip, i*64+ 6, parm); *((uint64_t *)op+i*63+ 6) = (uint64_t)IPW(ip, i*64+ 6) >> 6 | (uint64_t)IPX(ip, i*64+7) << 57;\
- IP64(ip, i*64+ 7, parm); *((uint64_t *)op+i*63+ 7) = (uint64_t)IPW(ip, i*64+ 7) >> 7 | (uint64_t)IPX(ip, i*64+8) << 56;\
- IP64(ip, i*64+ 8, parm); *((uint64_t *)op+i*63+ 8) = (uint64_t)IPW(ip, i*64+ 8) >> 8 | (uint64_t)IPX(ip, i*64+9) << 55;\
- IP64(ip, i*64+ 9, parm); *((uint64_t *)op+i*63+ 9) = (uint64_t)IPW(ip, i*64+ 9) >> 9 | (uint64_t)IPX(ip, i*64+10) << 54;\
- IP64(ip, i*64+10, parm); *((uint64_t *)op+i*63+10) = (uint64_t)IPW(ip, i*64+10) >> 10 | (uint64_t)IPX(ip, i*64+11) << 53;\
- IP64(ip, i*64+11, parm); *((uint64_t *)op+i*63+11) = (uint64_t)IPW(ip, i*64+11) >> 11 | (uint64_t)IPX(ip, i*64+12) << 52;\
- IP64(ip, i*64+12, parm); *((uint64_t *)op+i*63+12) = (uint64_t)IPW(ip, i*64+12) >> 12 | (uint64_t)IPX(ip, i*64+13) << 51;\
- IP64(ip, i*64+13, parm); *((uint64_t *)op+i*63+13) = (uint64_t)IPW(ip, i*64+13) >> 13 | (uint64_t)IPX(ip, i*64+14) << 50;\
- IP64(ip, i*64+14, parm); *((uint64_t *)op+i*63+14) = (uint64_t)IPW(ip, i*64+14) >> 14 | (uint64_t)IPX(ip, i*64+15) << 49;\
- IP64(ip, i*64+15, parm); *((uint64_t *)op+i*63+15) = (uint64_t)IPW(ip, i*64+15) >> 15 | (uint64_t)IPX(ip, i*64+16) << 48;\
- IP64(ip, i*64+16, parm); *((uint64_t *)op+i*63+16) = (uint64_t)IPW(ip, i*64+16) >> 16 | (uint64_t)IPX(ip, i*64+17) << 47;\
- IP64(ip, i*64+17, parm); *((uint64_t *)op+i*63+17) = (uint64_t)IPW(ip, i*64+17) >> 17 | (uint64_t)IPX(ip, i*64+18) << 46;\
- IP64(ip, i*64+18, parm); *((uint64_t *)op+i*63+18) = (uint64_t)IPW(ip, i*64+18) >> 18 | (uint64_t)IPX(ip, i*64+19) << 45;\
- IP64(ip, i*64+19, parm); *((uint64_t *)op+i*63+19) = (uint64_t)IPW(ip, i*64+19) >> 19 | (uint64_t)IPX(ip, i*64+20) << 44;\
- IP64(ip, i*64+20, parm); *((uint64_t *)op+i*63+20) = (uint64_t)IPW(ip, i*64+20) >> 20 | (uint64_t)IPX(ip, i*64+21) << 43;\
- IP64(ip, i*64+21, parm); *((uint64_t *)op+i*63+21) = (uint64_t)IPW(ip, i*64+21) >> 21 | (uint64_t)IPX(ip, i*64+22) << 42;\
- IP64(ip, i*64+22, parm); *((uint64_t *)op+i*63+22) = (uint64_t)IPW(ip, i*64+22) >> 22 | (uint64_t)IPX(ip, i*64+23) << 41;\
- IP64(ip, i*64+23, parm); *((uint64_t *)op+i*63+23) = (uint64_t)IPW(ip, i*64+23) >> 23 | (uint64_t)IPX(ip, i*64+24) << 40;\
- IP64(ip, i*64+24, parm); *((uint64_t *)op+i*63+24) = (uint64_t)IPW(ip, i*64+24) >> 24 | (uint64_t)IPX(ip, i*64+25) << 39;\
- IP64(ip, i*64+25, parm); *((uint64_t *)op+i*63+25) = (uint64_t)IPW(ip, i*64+25) >> 25 | (uint64_t)IPX(ip, i*64+26) << 38;\
- IP64(ip, i*64+26, parm); *((uint64_t *)op+i*63+26) = (uint64_t)IPW(ip, i*64+26) >> 26 | (uint64_t)IPX(ip, i*64+27) << 37;\
- IP64(ip, i*64+27, parm); *((uint64_t *)op+i*63+27) = (uint64_t)IPW(ip, i*64+27) >> 27 | (uint64_t)IPX(ip, i*64+28) << 36;\
- IP64(ip, i*64+28, parm); *((uint64_t *)op+i*63+28) = (uint64_t)IPW(ip, i*64+28) >> 28 | (uint64_t)IPX(ip, i*64+29) << 35;\
- IP64(ip, i*64+29, parm); *((uint64_t *)op+i*63+29) = (uint64_t)IPW(ip, i*64+29) >> 29 | (uint64_t)IPX(ip, i*64+30) << 34;\
- IP64(ip, i*64+30, parm); *((uint64_t *)op+i*63+30) = (uint64_t)IPW(ip, i*64+30) >> 30 | (uint64_t)IPX(ip, i*64+31) << 33;\
- IP64(ip, i*64+31, parm); *((uint64_t *)op+i*63+31) = (uint64_t)IPW(ip, i*64+31) >> 31;\
-}
-
-#define BITPACK64_63(ip, op, parm) { \
- BITBLK64_63(ip, 0, op, parm); IPI(ip); op += 63*4/sizeof(op[0]);\
-}
-
-#define BITBLK64_64(ip, i, op, parm) { ;\
- IP9(ip, i*1+ 0, parm); *((uint64_t *)op+i*1+ 0) = (uint64_t)IPV(ip, i*1+ 0) ;\
-}
-
-#define BITPACK64_64(ip, op, parm) { \
- BITBLK64_64(ip, 0, op, parm);\
- BITBLK64_64(ip, 1, op, parm);\
- BITBLK64_64(ip, 2, op, parm);\
- BITBLK64_64(ip, 3, op, parm);\
- BITBLK64_64(ip, 4, op, parm);\
- BITBLK64_64(ip, 5, op, parm);\
- BITBLK64_64(ip, 6, op, parm);\
- BITBLK64_64(ip, 7, op, parm);\
- BITBLK64_64(ip, 8, op, parm);\
- BITBLK64_64(ip, 9, op, parm);\
- BITBLK64_64(ip, 10, op, parm);\
- BITBLK64_64(ip, 11, op, parm);\
- BITBLK64_64(ip, 12, op, parm);\
- BITBLK64_64(ip, 13, op, parm);\
- BITBLK64_64(ip, 14, op, parm);\
- BITBLK64_64(ip, 15, op, parm);\
- BITBLK64_64(ip, 16, op, parm);\
- BITBLK64_64(ip, 17, op, parm);\
- BITBLK64_64(ip, 18, op, parm);\
- BITBLK64_64(ip, 19, op, parm);\
- BITBLK64_64(ip, 20, op, parm);\
- BITBLK64_64(ip, 21, op, parm);\
- BITBLK64_64(ip, 22, op, parm);\
- BITBLK64_64(ip, 23, op, parm);\
- BITBLK64_64(ip, 24, op, parm);\
- BITBLK64_64(ip, 25, op, parm);\
- BITBLK64_64(ip, 26, op, parm);\
- BITBLK64_64(ip, 27, op, parm);\
- BITBLK64_64(ip, 28, op, parm);\
- BITBLK64_64(ip, 29, op, parm);\
- BITBLK64_64(ip, 30, op, parm);\
- BITBLK64_64(ip, 31, op, parm); IPI(ip); op += 64*4/sizeof(op[0]);\
-}
-
-#ifndef DELTA
-#define USIZE 8
-unsigned char *TEMPLATE2(_BITPACK_,8_0)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,8_1)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint8_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_2)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint8_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_3)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint8_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_4)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint8_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_5)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint8_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_6)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint8_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_7)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint8_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_8)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint8_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F8 TEMPLATE2(_BITPACK_,a8)[] = {
- &TEMPLATE2(_BITPACK_,8_0),
- &TEMPLATE2(_BITPACK_,8_1),
- &TEMPLATE2(_BITPACK_,8_2),
- &TEMPLATE2(_BITPACK_,8_3),
- &TEMPLATE2(_BITPACK_,8_4),
- &TEMPLATE2(_BITPACK_,8_5),
- &TEMPLATE2(_BITPACK_,8_6),
- &TEMPLATE2(_BITPACK_,8_7),
- &TEMPLATE2(_BITPACK_,8_8)
-};
-unsigned char *TEMPLATE2(_BITPACK_,8)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a8)[ b](in, n, out); }
-
-#define USIZE 16
-unsigned char *TEMPLATE2(_BITPACK_,16_0)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,16_1)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint16_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_2)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint16_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_3)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint16_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_4)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint16_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_5)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint16_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_6)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint16_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_7)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint16_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_8)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint16_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_9)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint16_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_10)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint16_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_11)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint16_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_12)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint16_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_13)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint16_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_14)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint16_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_15)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint16_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_16)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint16_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F16 TEMPLATE2(_BITPACK_,a16)[] = {
- &TEMPLATE2(_BITPACK_,16_0),
- &TEMPLATE2(_BITPACK_,16_1),
- &TEMPLATE2(_BITPACK_,16_2),
- &TEMPLATE2(_BITPACK_,16_3),
- &TEMPLATE2(_BITPACK_,16_4),
- &TEMPLATE2(_BITPACK_,16_5),
- &TEMPLATE2(_BITPACK_,16_6),
- &TEMPLATE2(_BITPACK_,16_7),
- &TEMPLATE2(_BITPACK_,16_8),
- &TEMPLATE2(_BITPACK_,16_9),
- &TEMPLATE2(_BITPACK_,16_10),
- &TEMPLATE2(_BITPACK_,16_11),
- &TEMPLATE2(_BITPACK_,16_12),
- &TEMPLATE2(_BITPACK_,16_13),
- &TEMPLATE2(_BITPACK_,16_14),
- &TEMPLATE2(_BITPACK_,16_15),
- &TEMPLATE2(_BITPACK_,16_16)
-};
-unsigned char *TEMPLATE2(_BITPACK_,16)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a16)[ b](in, n, out); }
-
-#define USIZE 32
-unsigned char *TEMPLATE2(_BITPACK_,32_0)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,32_1)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint32_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_2)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint32_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_3)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint32_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_4)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint32_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_5)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint32_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_6)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint32_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_7)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint32_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_8)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint32_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_9)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint32_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_10)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint32_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_11)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint32_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_12)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint32_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_13)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint32_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_14)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint32_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_15)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint32_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_16)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint32_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_17)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*17); uint32_t v,x;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_18)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*18); uint32_t v,x;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_19)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*19); uint32_t v,x;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_20)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*20); uint32_t v,x;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_21)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*21); uint32_t v,x;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_22)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*22); uint32_t v,x;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_23)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*23); uint32_t v,x;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_24)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*24); uint32_t v,x;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_25)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*25); uint32_t v,x;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_26)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*26); uint32_t v,x;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_27)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*27); uint32_t v,x;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_28)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*28); uint32_t v,x;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_29)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*29); uint32_t v,x;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_30)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*30); uint32_t v,x;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_31)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*31); uint32_t v,x;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_32)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*32); uint32_t v,x;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F32 TEMPLATE2(_BITPACK_,a32)[] = {
- &TEMPLATE2(_BITPACK_,32_0),
- &TEMPLATE2(_BITPACK_,32_1),
- &TEMPLATE2(_BITPACK_,32_2),
- &TEMPLATE2(_BITPACK_,32_3),
- &TEMPLATE2(_BITPACK_,32_4),
- &TEMPLATE2(_BITPACK_,32_5),
- &TEMPLATE2(_BITPACK_,32_6),
- &TEMPLATE2(_BITPACK_,32_7),
- &TEMPLATE2(_BITPACK_,32_8),
- &TEMPLATE2(_BITPACK_,32_9),
- &TEMPLATE2(_BITPACK_,32_10),
- &TEMPLATE2(_BITPACK_,32_11),
- &TEMPLATE2(_BITPACK_,32_12),
- &TEMPLATE2(_BITPACK_,32_13),
- &TEMPLATE2(_BITPACK_,32_14),
- &TEMPLATE2(_BITPACK_,32_15),
- &TEMPLATE2(_BITPACK_,32_16),
- &TEMPLATE2(_BITPACK_,32_17),
- &TEMPLATE2(_BITPACK_,32_18),
- &TEMPLATE2(_BITPACK_,32_19),
- &TEMPLATE2(_BITPACK_,32_20),
- &TEMPLATE2(_BITPACK_,32_21),
- &TEMPLATE2(_BITPACK_,32_22),
- &TEMPLATE2(_BITPACK_,32_23),
- &TEMPLATE2(_BITPACK_,32_24),
- &TEMPLATE2(_BITPACK_,32_25),
- &TEMPLATE2(_BITPACK_,32_26),
- &TEMPLATE2(_BITPACK_,32_27),
- &TEMPLATE2(_BITPACK_,32_28),
- &TEMPLATE2(_BITPACK_,32_29),
- &TEMPLATE2(_BITPACK_,32_30),
- &TEMPLATE2(_BITPACK_,32_31),
- &TEMPLATE2(_BITPACK_,32_32)
-};
-unsigned char *TEMPLATE2(_BITPACK_,32)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a32)[ b](in, n, out); }
-
-#define USIZE 64
-unsigned char *TEMPLATE2(_BITPACK_,64_0)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,64_1)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*1); uint64_t v,x;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_2)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*2); uint64_t v,x;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_3)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*3); uint64_t v,x;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_4)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*4); uint64_t v,x;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_5)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*5); uint64_t v,x;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_6)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*6); uint64_t v,x;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_7)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*7); uint64_t v,x;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_8)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*8); uint64_t v,x;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_9)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*9); uint64_t v,x;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_10)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*10); uint64_t v,x;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_11)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*11); uint64_t v,x;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_12)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*12); uint64_t v,x;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_13)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*13); uint64_t v,x;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_14)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*14); uint64_t v,x;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_15)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*15); uint64_t v,x;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_16)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*16); uint64_t v,x;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_17)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*17); uint64_t v,x;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_18)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*18); uint64_t v,x;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_19)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*19); uint64_t v,x;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_20)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*20); uint64_t v,x;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_21)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*21); uint64_t v,x;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_22)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*22); uint64_t v,x;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_23)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*23); uint64_t v,x;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_24)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*24); uint64_t v,x;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_25)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*25); uint64_t v,x;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_26)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*26); uint64_t v,x;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_27)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*27); uint64_t v,x;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_28)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*28); uint64_t v,x;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_29)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*29); uint64_t v,x;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_30)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*30); uint64_t v,x;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_31)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*31); uint64_t v,x;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_32)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*32); uint64_t v,x;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_33)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*33); uint64_t v,x;do { BITPACK64_33( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_34)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*34); uint64_t v,x;do { BITPACK64_34( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_35)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*35); uint64_t v,x;do { BITPACK64_35( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_36)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*36); uint64_t v,x;do { BITPACK64_36( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_37)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*37); uint64_t v,x;do { BITPACK64_37( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_38)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*38); uint64_t v,x;do { BITPACK64_38( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_39)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*39); uint64_t v,x;do { BITPACK64_39( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_40)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*40); uint64_t v,x;do { BITPACK64_40( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_41)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*41); uint64_t v,x;do { BITPACK64_41( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_42)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*42); uint64_t v,x;do { BITPACK64_42( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_43)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*43); uint64_t v,x;do { BITPACK64_43( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_44)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*44); uint64_t v,x;do { BITPACK64_44( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_45)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*45); uint64_t v,x;do { BITPACK64_45( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_46)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*46); uint64_t v,x;do { BITPACK64_46( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_47)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*47); uint64_t v,x;do { BITPACK64_47( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_48)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*48); uint64_t v,x;do { BITPACK64_48( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_49)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*49); uint64_t v,x;do { BITPACK64_49( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_50)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*50); uint64_t v,x;do { BITPACK64_50( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_51)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*51); uint64_t v,x;do { BITPACK64_51( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_52)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*52); uint64_t v,x;do { BITPACK64_52( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_53)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*53); uint64_t v,x;do { BITPACK64_53( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_54)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*54); uint64_t v,x;do { BITPACK64_54( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_55)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*55); uint64_t v,x;do { BITPACK64_55( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_56)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*56); uint64_t v,x;do { BITPACK64_56( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_57)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*57); uint64_t v,x;do { BITPACK64_57( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_58)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*58); uint64_t v,x;do { BITPACK64_58( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_59)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*59); uint64_t v,x;do { BITPACK64_59( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_60)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*60); uint64_t v,x;do { BITPACK64_60( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_61)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*61); uint64_t v,x;do { BITPACK64_61( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_62)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*62); uint64_t v,x;do { BITPACK64_62( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_63)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*63); uint64_t v,x;do { BITPACK64_63( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_64)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out ) { unsigned char *out_=out+PAD8(n*64); uint64_t v,x;do { BITPACK64_64( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_F64 TEMPLATE2(_BITPACK_,a64)[] = {
- &TEMPLATE2(_BITPACK_,64_0),
- &TEMPLATE2(_BITPACK_,64_1),
- &TEMPLATE2(_BITPACK_,64_2),
- &TEMPLATE2(_BITPACK_,64_3),
- &TEMPLATE2(_BITPACK_,64_4),
- &TEMPLATE2(_BITPACK_,64_5),
- &TEMPLATE2(_BITPACK_,64_6),
- &TEMPLATE2(_BITPACK_,64_7),
- &TEMPLATE2(_BITPACK_,64_8),
- &TEMPLATE2(_BITPACK_,64_9),
- &TEMPLATE2(_BITPACK_,64_10),
- &TEMPLATE2(_BITPACK_,64_11),
- &TEMPLATE2(_BITPACK_,64_12),
- &TEMPLATE2(_BITPACK_,64_13),
- &TEMPLATE2(_BITPACK_,64_14),
- &TEMPLATE2(_BITPACK_,64_15),
- &TEMPLATE2(_BITPACK_,64_16),
- &TEMPLATE2(_BITPACK_,64_17),
- &TEMPLATE2(_BITPACK_,64_18),
- &TEMPLATE2(_BITPACK_,64_19),
- &TEMPLATE2(_BITPACK_,64_20),
- &TEMPLATE2(_BITPACK_,64_21),
- &TEMPLATE2(_BITPACK_,64_22),
- &TEMPLATE2(_BITPACK_,64_23),
- &TEMPLATE2(_BITPACK_,64_24),
- &TEMPLATE2(_BITPACK_,64_25),
- &TEMPLATE2(_BITPACK_,64_26),
- &TEMPLATE2(_BITPACK_,64_27),
- &TEMPLATE2(_BITPACK_,64_28),
- &TEMPLATE2(_BITPACK_,64_29),
- &TEMPLATE2(_BITPACK_,64_30),
- &TEMPLATE2(_BITPACK_,64_31),
- &TEMPLATE2(_BITPACK_,64_32),
- &TEMPLATE2(_BITPACK_,64_33),
- &TEMPLATE2(_BITPACK_,64_34),
- &TEMPLATE2(_BITPACK_,64_35),
- &TEMPLATE2(_BITPACK_,64_36),
- &TEMPLATE2(_BITPACK_,64_37),
- &TEMPLATE2(_BITPACK_,64_38),
- &TEMPLATE2(_BITPACK_,64_39),
- &TEMPLATE2(_BITPACK_,64_40),
- &TEMPLATE2(_BITPACK_,64_41),
- &TEMPLATE2(_BITPACK_,64_42),
- &TEMPLATE2(_BITPACK_,64_43),
- &TEMPLATE2(_BITPACK_,64_44),
- &TEMPLATE2(_BITPACK_,64_45),
- &TEMPLATE2(_BITPACK_,64_46),
- &TEMPLATE2(_BITPACK_,64_47),
- &TEMPLATE2(_BITPACK_,64_48),
- &TEMPLATE2(_BITPACK_,64_49),
- &TEMPLATE2(_BITPACK_,64_50),
- &TEMPLATE2(_BITPACK_,64_51),
- &TEMPLATE2(_BITPACK_,64_52),
- &TEMPLATE2(_BITPACK_,64_53),
- &TEMPLATE2(_BITPACK_,64_54),
- &TEMPLATE2(_BITPACK_,64_55),
- &TEMPLATE2(_BITPACK_,64_56),
- &TEMPLATE2(_BITPACK_,64_57),
- &TEMPLATE2(_BITPACK_,64_58),
- &TEMPLATE2(_BITPACK_,64_59),
- &TEMPLATE2(_BITPACK_,64_60),
- &TEMPLATE2(_BITPACK_,64_61),
- &TEMPLATE2(_BITPACK_,64_62),
- &TEMPLATE2(_BITPACK_,64_63),
- &TEMPLATE2(_BITPACK_,64_64)
-};
-unsigned char *TEMPLATE2(_BITPACK_,64)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , unsigned b) { return TEMPLATE2(_BITPACK_,a64)[ b](in, n, out); }
-
-#else
-#define USIZE 8
-unsigned char *TEMPLATE2(_BITPACK_,8_0)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,8_1)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*1); uint8_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_2)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*2); uint8_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_3)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*3); uint8_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_4)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*4); uint8_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_5)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*5); uint8_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_6)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*6); uint8_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_7)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*7); uint8_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,8_8)(uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start ) { unsigned char *out_=out+PAD8(n*8); uint8_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D8 TEMPLATE2(_BITPACK_,a8)[] = {
- &TEMPLATE2(_BITPACK_,8_0),
- &TEMPLATE2(_BITPACK_,8_1),
- &TEMPLATE2(_BITPACK_,8_2),
- &TEMPLATE2(_BITPACK_,8_3),
- &TEMPLATE2(_BITPACK_,8_4),
- &TEMPLATE2(_BITPACK_,8_5),
- &TEMPLATE2(_BITPACK_,8_6),
- &TEMPLATE2(_BITPACK_,8_7),
- &TEMPLATE2(_BITPACK_,8_8)
-};
-unsigned char *TEMPLATE2(_BITPACK_,8)( uint8_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint8_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a8)[ b](in, n, out, start); }
-
-#define USIZE 16
-unsigned char *TEMPLATE2(_BITPACK_,16_0)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,16_1)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*1); uint16_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_2)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*2); uint16_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_3)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*3); uint16_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_4)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*4); uint16_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_5)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*5); uint16_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_6)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*6); uint16_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_7)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*7); uint16_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_8)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*8); uint16_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_9)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*9); uint16_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_10)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*10); uint16_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_11)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*11); uint16_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_12)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*12); uint16_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_13)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*13); uint16_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_14)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*14); uint16_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_15)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*15); uint16_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,16_16)(uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start ) { unsigned char *out_=out+PAD8(n*16); uint16_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D16 TEMPLATE2(_BITPACK_,a16)[] = {
- &TEMPLATE2(_BITPACK_,16_0),
- &TEMPLATE2(_BITPACK_,16_1),
- &TEMPLATE2(_BITPACK_,16_2),
- &TEMPLATE2(_BITPACK_,16_3),
- &TEMPLATE2(_BITPACK_,16_4),
- &TEMPLATE2(_BITPACK_,16_5),
- &TEMPLATE2(_BITPACK_,16_6),
- &TEMPLATE2(_BITPACK_,16_7),
- &TEMPLATE2(_BITPACK_,16_8),
- &TEMPLATE2(_BITPACK_,16_9),
- &TEMPLATE2(_BITPACK_,16_10),
- &TEMPLATE2(_BITPACK_,16_11),
- &TEMPLATE2(_BITPACK_,16_12),
- &TEMPLATE2(_BITPACK_,16_13),
- &TEMPLATE2(_BITPACK_,16_14),
- &TEMPLATE2(_BITPACK_,16_15),
- &TEMPLATE2(_BITPACK_,16_16)
-};
-unsigned char *TEMPLATE2(_BITPACK_,16)( uint16_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint16_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a16)[ b](in, n, out, start); }
-
-#define USIZE 32
-unsigned char *TEMPLATE2(_BITPACK_,32_0)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,32_1)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*1); uint32_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_2)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*2); uint32_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_3)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*3); uint32_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_4)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*4); uint32_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_5)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*5); uint32_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_6)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*6); uint32_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_7)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*7); uint32_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_8)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*8); uint32_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_9)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*9); uint32_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_10)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*10); uint32_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_11)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*11); uint32_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_12)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*12); uint32_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_13)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*13); uint32_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_14)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*14); uint32_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_15)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*15); uint32_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_16)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*16); uint32_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_17)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*17); uint32_t v,x=0;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_18)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*18); uint32_t v,x=0;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_19)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*19); uint32_t v,x=0;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_20)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*20); uint32_t v,x=0;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_21)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*21); uint32_t v,x=0;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_22)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*22); uint32_t v,x=0;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_23)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*23); uint32_t v,x=0;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_24)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*24); uint32_t v,x=0;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_25)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*25); uint32_t v,x=0;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_26)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*26); uint32_t v,x=0;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_27)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*27); uint32_t v,x=0;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_28)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*28); uint32_t v,x=0;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_29)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*29); uint32_t v,x=0;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_30)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*30); uint32_t v,x=0;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_31)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*31); uint32_t v,x=0;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,32_32)(uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start ) { unsigned char *out_=out+PAD8(n*32); uint32_t v,x=0;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D32 TEMPLATE2(_BITPACK_,a32)[] = {
- &TEMPLATE2(_BITPACK_,32_0),
- &TEMPLATE2(_BITPACK_,32_1),
- &TEMPLATE2(_BITPACK_,32_2),
- &TEMPLATE2(_BITPACK_,32_3),
- &TEMPLATE2(_BITPACK_,32_4),
- &TEMPLATE2(_BITPACK_,32_5),
- &TEMPLATE2(_BITPACK_,32_6),
- &TEMPLATE2(_BITPACK_,32_7),
- &TEMPLATE2(_BITPACK_,32_8),
- &TEMPLATE2(_BITPACK_,32_9),
- &TEMPLATE2(_BITPACK_,32_10),
- &TEMPLATE2(_BITPACK_,32_11),
- &TEMPLATE2(_BITPACK_,32_12),
- &TEMPLATE2(_BITPACK_,32_13),
- &TEMPLATE2(_BITPACK_,32_14),
- &TEMPLATE2(_BITPACK_,32_15),
- &TEMPLATE2(_BITPACK_,32_16),
- &TEMPLATE2(_BITPACK_,32_17),
- &TEMPLATE2(_BITPACK_,32_18),
- &TEMPLATE2(_BITPACK_,32_19),
- &TEMPLATE2(_BITPACK_,32_20),
- &TEMPLATE2(_BITPACK_,32_21),
- &TEMPLATE2(_BITPACK_,32_22),
- &TEMPLATE2(_BITPACK_,32_23),
- &TEMPLATE2(_BITPACK_,32_24),
- &TEMPLATE2(_BITPACK_,32_25),
- &TEMPLATE2(_BITPACK_,32_26),
- &TEMPLATE2(_BITPACK_,32_27),
- &TEMPLATE2(_BITPACK_,32_28),
- &TEMPLATE2(_BITPACK_,32_29),
- &TEMPLATE2(_BITPACK_,32_30),
- &TEMPLATE2(_BITPACK_,32_31),
- &TEMPLATE2(_BITPACK_,32_32)
-};
-unsigned char *TEMPLATE2(_BITPACK_,32)( uint32_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint32_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a32)[ b](in, n, out, start); }
-
-#define USIZE 64
-unsigned char *TEMPLATE2(_BITPACK_,64_0)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { return out; }
-unsigned char *TEMPLATE2(_BITPACK_,64_1)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*1); uint64_t v,x=0;do { BITPACK64_1( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_2)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*2); uint64_t v,x=0;do { BITPACK64_2( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_3)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*3); uint64_t v,x=0;do { BITPACK64_3( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_4)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*4); uint64_t v,x=0;do { BITPACK64_4( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_5)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*5); uint64_t v,x=0;do { BITPACK64_5( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_6)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*6); uint64_t v,x=0;do { BITPACK64_6( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_7)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*7); uint64_t v,x=0;do { BITPACK64_7( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_8)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*8); uint64_t v,x=0;do { BITPACK64_8( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_9)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*9); uint64_t v,x=0;do { BITPACK64_9( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_10)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*10); uint64_t v,x=0;do { BITPACK64_10( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_11)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*11); uint64_t v,x=0;do { BITPACK64_11( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_12)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*12); uint64_t v,x=0;do { BITPACK64_12( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_13)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*13); uint64_t v,x=0;do { BITPACK64_13( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_14)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*14); uint64_t v,x=0;do { BITPACK64_14( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_15)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*15); uint64_t v,x=0;do { BITPACK64_15( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_16)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*16); uint64_t v,x=0;do { BITPACK64_16( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_17)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*17); uint64_t v,x=0;do { BITPACK64_17( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_18)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*18); uint64_t v,x=0;do { BITPACK64_18( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_19)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*19); uint64_t v,x=0;do { BITPACK64_19( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_20)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*20); uint64_t v,x=0;do { BITPACK64_20( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_21)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*21); uint64_t v,x=0;do { BITPACK64_21( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_22)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*22); uint64_t v,x=0;do { BITPACK64_22( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_23)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*23); uint64_t v,x=0;do { BITPACK64_23( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_24)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*24); uint64_t v,x=0;do { BITPACK64_24( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_25)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*25); uint64_t v,x=0;do { BITPACK64_25( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_26)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*26); uint64_t v,x=0;do { BITPACK64_26( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_27)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*27); uint64_t v,x=0;do { BITPACK64_27( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_28)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*28); uint64_t v,x=0;do { BITPACK64_28( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_29)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*29); uint64_t v,x=0;do { BITPACK64_29( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_30)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*30); uint64_t v,x=0;do { BITPACK64_30( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_31)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*31); uint64_t v,x=0;do { BITPACK64_31( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_32)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*32); uint64_t v,x=0;do { BITPACK64_32( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_33)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*33); uint64_t v,x=0;do { BITPACK64_33( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_34)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*34); uint64_t v,x=0;do { BITPACK64_34( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_35)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*35); uint64_t v,x=0;do { BITPACK64_35( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_36)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*36); uint64_t v,x=0;do { BITPACK64_36( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_37)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*37); uint64_t v,x=0;do { BITPACK64_37( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_38)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*38); uint64_t v,x=0;do { BITPACK64_38( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_39)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*39); uint64_t v,x=0;do { BITPACK64_39( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_40)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*40); uint64_t v,x=0;do { BITPACK64_40( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_41)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*41); uint64_t v,x=0;do { BITPACK64_41( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_42)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*42); uint64_t v,x=0;do { BITPACK64_42( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_43)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*43); uint64_t v,x=0;do { BITPACK64_43( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_44)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*44); uint64_t v,x=0;do { BITPACK64_44( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_45)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*45); uint64_t v,x=0;do { BITPACK64_45( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_46)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*46); uint64_t v,x=0;do { BITPACK64_46( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_47)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*47); uint64_t v,x=0;do { BITPACK64_47( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_48)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*48); uint64_t v,x=0;do { BITPACK64_48( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_49)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*49); uint64_t v,x=0;do { BITPACK64_49( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_50)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*50); uint64_t v,x=0;do { BITPACK64_50( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_51)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*51); uint64_t v,x=0;do { BITPACK64_51( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_52)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*52); uint64_t v,x=0;do { BITPACK64_52( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_53)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*53); uint64_t v,x=0;do { BITPACK64_53( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_54)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*54); uint64_t v,x=0;do { BITPACK64_54( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_55)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*55); uint64_t v,x=0;do { BITPACK64_55( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_56)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*56); uint64_t v,x=0;do { BITPACK64_56( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_57)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*57); uint64_t v,x=0;do { BITPACK64_57( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_58)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*58); uint64_t v,x=0;do { BITPACK64_58( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_59)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*59); uint64_t v,x=0;do { BITPACK64_59( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_60)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*60); uint64_t v,x=0;do { BITPACK64_60( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_61)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*61); uint64_t v,x=0;do { BITPACK64_61( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_62)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*62); uint64_t v,x=0;do { BITPACK64_62( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_63)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*63); uint64_t v,x=0;do { BITPACK64_63( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-unsigned char *TEMPLATE2(_BITPACK_,64_64)(uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start ) { unsigned char *out_=out+PAD8(n*64); uint64_t v,x=0;do { BITPACK64_64( in, out, start); PREFETCH(in+512,0); } while(out<out_); return out_; }
-BITPACK_D64 TEMPLATE2(_BITPACK_,a64)[] = {
- &TEMPLATE2(_BITPACK_,64_0),
- &TEMPLATE2(_BITPACK_,64_1),
- &TEMPLATE2(_BITPACK_,64_2),
- &TEMPLATE2(_BITPACK_,64_3),
- &TEMPLATE2(_BITPACK_,64_4),
- &TEMPLATE2(_BITPACK_,64_5),
- &TEMPLATE2(_BITPACK_,64_6),
- &TEMPLATE2(_BITPACK_,64_7),
- &TEMPLATE2(_BITPACK_,64_8),
- &TEMPLATE2(_BITPACK_,64_9),
- &TEMPLATE2(_BITPACK_,64_10),
- &TEMPLATE2(_BITPACK_,64_11),
- &TEMPLATE2(_BITPACK_,64_12),
- &TEMPLATE2(_BITPACK_,64_13),
- &TEMPLATE2(_BITPACK_,64_14),
- &TEMPLATE2(_BITPACK_,64_15),
- &TEMPLATE2(_BITPACK_,64_16),
- &TEMPLATE2(_BITPACK_,64_17),
- &TEMPLATE2(_BITPACK_,64_18),
- &TEMPLATE2(_BITPACK_,64_19),
- &TEMPLATE2(_BITPACK_,64_20),
- &TEMPLATE2(_BITPACK_,64_21),
- &TEMPLATE2(_BITPACK_,64_22),
- &TEMPLATE2(_BITPACK_,64_23),
- &TEMPLATE2(_BITPACK_,64_24),
- &TEMPLATE2(_BITPACK_,64_25),
- &TEMPLATE2(_BITPACK_,64_26),
- &TEMPLATE2(_BITPACK_,64_27),
- &TEMPLATE2(_BITPACK_,64_28),
- &TEMPLATE2(_BITPACK_,64_29),
- &TEMPLATE2(_BITPACK_,64_30),
- &TEMPLATE2(_BITPACK_,64_31),
- &TEMPLATE2(_BITPACK_,64_32),
- &TEMPLATE2(_BITPACK_,64_33),
- &TEMPLATE2(_BITPACK_,64_34),
- &TEMPLATE2(_BITPACK_,64_35),
- &TEMPLATE2(_BITPACK_,64_36),
- &TEMPLATE2(_BITPACK_,64_37),
- &TEMPLATE2(_BITPACK_,64_38),
- &TEMPLATE2(_BITPACK_,64_39),
- &TEMPLATE2(_BITPACK_,64_40),
- &TEMPLATE2(_BITPACK_,64_41),
- &TEMPLATE2(_BITPACK_,64_42),
- &TEMPLATE2(_BITPACK_,64_43),
- &TEMPLATE2(_BITPACK_,64_44),
- &TEMPLATE2(_BITPACK_,64_45),
- &TEMPLATE2(_BITPACK_,64_46),
- &TEMPLATE2(_BITPACK_,64_47),
- &TEMPLATE2(_BITPACK_,64_48),
- &TEMPLATE2(_BITPACK_,64_49),
- &TEMPLATE2(_BITPACK_,64_50),
- &TEMPLATE2(_BITPACK_,64_51),
- &TEMPLATE2(_BITPACK_,64_52),
- &TEMPLATE2(_BITPACK_,64_53),
- &TEMPLATE2(_BITPACK_,64_54),
- &TEMPLATE2(_BITPACK_,64_55),
- &TEMPLATE2(_BITPACK_,64_56),
- &TEMPLATE2(_BITPACK_,64_57),
- &TEMPLATE2(_BITPACK_,64_58),
- &TEMPLATE2(_BITPACK_,64_59),
- &TEMPLATE2(_BITPACK_,64_60),
- &TEMPLATE2(_BITPACK_,64_61),
- &TEMPLATE2(_BITPACK_,64_62),
- &TEMPLATE2(_BITPACK_,64_63),
- &TEMPLATE2(_BITPACK_,64_64)
-};
-unsigned char *TEMPLATE2(_BITPACK_,64)( uint64_t *__restrict in, unsigned n, const unsigned char *__restrict out , uint64_t start, unsigned b) { return TEMPLATE2(_BITPACK_,a64)[ b](in, n, out, start); }
-
-#endif
-#endif //IP9
-
-//---------------------------------------------------------
-#define BITBLK128V16_1(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 1));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 2));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 3));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 5));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 6));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 7));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 9));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 10));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 11));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 12));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 13));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 14));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 15)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_1(ip, op, parm) {\
- BITBLK128V16_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_2(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 1, iv), 2));\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 2, iv), 4));\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 3, iv), 6));\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 4, iv), 8));\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 5, iv), 10));\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 6, iv), 12));\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 14)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_2(ip, op, parm) {\
- BITBLK128V16_2(ip, 0, op, parm);\
- BITBLK128V16_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_3(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 3));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 6));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 9));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 12));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 2));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 5));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 11));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 1));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 7));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 10));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 13)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_3(ip, op, parm) {\
- BITBLK128V16_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_4(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*4+ 0, iv, parm); ov = IP16(ip, i*4+ 0, iv);\
- VI16(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 1, iv), 4));\
- VI16(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 2, iv), 8));\
- VI16(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 3, iv), 12)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_4(ip, op, parm) {\
- BITBLK128V16_4(ip, 0, op, parm);\
- BITBLK128V16_4(ip, 1, op, parm);\
- BITBLK128V16_4(ip, 2, op, parm);\
- BITBLK128V16_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_5(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 5));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 10));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 9));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 3));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 2));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 7));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 1));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 6));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 11)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_5(ip, op, parm) {\
- BITBLK128V16_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_6(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 1, iv), 6));\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 3, iv), 2));\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 4, iv), 8));\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 5, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 6, iv), 4));\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 10)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_6(ip, op, parm) {\
- BITBLK128V16_6(ip, 0, op, parm);\
- BITBLK128V16_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_7(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 1, iv), 7));\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 5));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 3));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 7, iv), 1));\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 8, iv), 8));\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 6));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+14, iv), 2));\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 9)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_7(ip, op, parm) {\
- BITBLK128V16_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_8(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*2+ 0, iv, parm); ov = IP16(ip, i*2+ 0, iv);\
- VI16(ip, i*2+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*2+ 1, iv), 8)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_8(ip, op, parm) {\
- BITBLK128V16_8(ip, 0, op, parm);\
- BITBLK128V16_8(ip, 1, op, parm);\
- BITBLK128V16_8(ip, 2, op, parm);\
- BITBLK128V16_8(ip, 3, op, parm);\
- BITBLK128V16_8(ip, 4, op, parm);\
- BITBLK128V16_8(ip, 5, op, parm);\
- BITBLK128V16_8(ip, 6, op, parm);\
- BITBLK128V16_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_9(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 2, iv), 2));\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 4, iv), 4));\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 6));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 1));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+11, iv), 3));\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+13, iv), 5));\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 7)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_9(ip, op, parm) {\
- BITBLK128V16_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_10(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 1, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 2, iv), 4));\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 3, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 5, iv), 2));\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 6, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 6)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_10(ip, op, parm) {\
- BITBLK128V16_10(ip, 0, op, parm);\
- BITBLK128V16_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_11(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 3, iv), 1));\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 6, iv), 2));\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 9, iv), 3));\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+12, iv), 4));\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 5)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_11(ip, op, parm) {\
- BITBLK128V16_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_12(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*4+ 0, iv, parm); ov = IP16(ip, i*4+ 0, iv);\
- VI16(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*4+ 1, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*4+ 2, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*4+ 3, iv), 4)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_12(ip, op, parm) {\
- BITBLK128V16_12(ip, 0, op, parm);\
- BITBLK128V16_12(ip, 1, op, parm);\
- BITBLK128V16_12(ip, 2, op, parm);\
- BITBLK128V16_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_13(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+ 5, iv), 1));\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 11);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+10, iv), 2));\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 3)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_13(ip, op, parm) {\
- BITBLK128V16_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_14(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*8+ 0, iv, parm); ov = IP16(ip, i*8+ 0, iv);\
- VI16(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 1, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 3, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 5, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*8+ 6, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*8+ 7, iv), 2)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_14(ip, op, parm) {\
- BITBLK128V16_14(ip, 0, op, parm);\
- BITBLK128V16_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_15(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*16+ 0, iv, parm); ov = IP16(ip, i*16+ 0, iv);\
- VI16(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 1, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 1);\
- VI16(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 2);\
- VI16(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 3, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 3);\
- VI16(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 4);\
- VI16(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 5, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 5);\
- VI16(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 6);\
- VI16(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 7, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 7);\
- VI16(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 8);\
- VI16(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+ 9, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 9);\
- VI16(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+10, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 10);\
- VI16(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+11, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 11);\
- VI16(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+12, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 12);\
- VI16(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+13, iv), 3)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 13);\
- VI16(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16(iv = IP16(ip, i*16+14, iv), 2)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi16(iv, 14);\
- VI16(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi16( IP16(ip, i*16+15, iv), 1)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_15(ip, op, parm) {\
- BITBLK128V16_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V16_16(ip, i, op, parm) { __m128i ov,iv;\
- VI16(ip, i*1+ 0, iv, parm); ov = IP16(ip, i*1+ 0, iv); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V16_16(ip, op, parm) {\
- BITBLK128V16_16(ip, 0, op, parm);\
- BITBLK128V16_16(ip, 1, op, parm);\
- BITBLK128V16_16(ip, 2, op, parm);\
- BITBLK128V16_16(ip, 3, op, parm);\
- BITBLK128V16_16(ip, 4, op, parm);\
- BITBLK128V16_16(ip, 5, op, parm);\
- BITBLK128V16_16(ip, 6, op, parm);\
- BITBLK128V16_16(ip, 7, op, parm);\
- BITBLK128V16_16(ip, 8, op, parm);\
- BITBLK128V16_16(ip, 9, op, parm);\
- BITBLK128V16_16(ip, 10, op, parm);\
- BITBLK128V16_16(ip, 11, op, parm);\
- BITBLK128V16_16(ip, 12, op, parm);\
- BITBLK128V16_16(ip, 13, op, parm);\
- BITBLK128V16_16(ip, 14, op, parm);\
- BITBLK128V16_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_1(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 1));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 3));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 5));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 7));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 9));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 11));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 13));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 15));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 17));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 18));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 19));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 21));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 22));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 23));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 25));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 26));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 27));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 28));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 29));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 30));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 31)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_1(ip, op, parm) {\
- BITBLK128V32_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_2(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 2));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 6));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 10));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 14));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 18));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 20));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 22));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 24));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 26));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 28));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 30)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_2(ip, op, parm) {\
- BITBLK128V32_2(ip, 0, op, parm);\
- BITBLK128V32_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_3(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 3));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 9));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 15));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 18));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 21));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 27));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 1));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 7));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 13));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 19));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 22));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 25));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 28));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 5));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 11));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 14));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 17));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 23));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 26));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 29)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_3(ip, op, parm) {\
- BITBLK128V32_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_4(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 1, iv), 4));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 3, iv), 12));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 5, iv), 20));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 6, iv), 24));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 28)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_4(ip, op, parm) {\
- BITBLK128V32_4(ip, 0, op, parm);\
- BITBLK128V32_4(ip, 1, op, parm);\
- BITBLK128V32_4(ip, 2, op, parm);\
- BITBLK128V32_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_5(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 5));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 15));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 20));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 25));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 3));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 13));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 18));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 23));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 1));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 11));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 21));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 26));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 9));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 14));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 19));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 7));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 17));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 22));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 27)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_5(ip, op, parm) {\
- BITBLK128V32_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_6(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 6));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 12));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 18));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 24));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 10));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 22));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 2));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 14));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 20));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 26)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_6(ip, op, parm) {\
- BITBLK128V32_6(ip, 0, op, parm);\
- BITBLK128V32_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_7(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 7));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 14));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 21));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 3));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 10));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 17));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 13));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 20));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 9));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 23));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 5));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 19));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 1));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 15));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 22));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 11));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 18));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 25)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_7(ip, op, parm) {\
- BITBLK128V32_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_8(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 1, iv), 8));\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 2, iv), 16));\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 3, iv), 24)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_8(ip, op, parm) {\
- BITBLK128V32_8(ip, 0, op, parm);\
- BITBLK128V32_8(ip, 1, op, parm);\
- BITBLK128V32_8(ip, 2, op, parm);\
- BITBLK128V32_8(ip, 3, op, parm);\
- BITBLK128V32_8(ip, 4, op, parm);\
- BITBLK128V32_8(ip, 5, op, parm);\
- BITBLK128V32_8(ip, 6, op, parm);\
- BITBLK128V32_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_9(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 9));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 18));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 13));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 22));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 17));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 3));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 21));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 7));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 11));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 15));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 1));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 10));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 19));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 5));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 14));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 23)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_9(ip, op, parm) {\
- BITBLK128V32_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_10(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 10));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 20));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 18));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 6));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 14));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 2));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 12));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 22)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_10(ip, op, parm) {\
- BITBLK128V32_10(ip, 0, op, parm);\
- BITBLK128V32_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_11(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 11));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 1));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 13));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 3));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 14));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 15));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 5));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 6));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 17));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 7));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 18));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 19));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 9));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 10));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 21)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_11(ip, op, parm) {\
- BITBLK128V32_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_12(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 1, iv), 12));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 3, iv), 4));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 6, iv), 8));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 20)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_12(ip, op, parm) {\
- BITBLK128V32_12(ip, 0, op, parm);\
- BITBLK128V32_12(ip, 1, op, parm);\
- BITBLK128V32_12(ip, 2, op, parm);\
- BITBLK128V32_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_13(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 13));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 7));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 1));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 14));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 15));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 9));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 3));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 10));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 17));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 11));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 5));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 18));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 6));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 19)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_13(ip, op, parm) {\
- BITBLK128V32_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_14(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 1, iv), 14));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 10));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 6));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 7, iv), 2));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 12));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+14, iv), 4));\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 18)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_14(ip, op, parm) {\
- BITBLK128V32_14(ip, 0, op, parm);\
- BITBLK128V32_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_15(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 1, iv), 15));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 13));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 11));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 9));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 7));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 5));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 3));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+15, iv), 1));\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 14));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 10));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 6));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+30, iv), 2));\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 17)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_15(ip, op, parm) {\
- BITBLK128V32_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_16(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*2+ 0, iv, parm); ov = IP32(ip, i*2+ 0, iv);\
- VI32(ip, i*2+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*2+ 1, iv), 16)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_16(ip, op, parm) {\
- BITBLK128V32_16(ip, 0, op, parm);\
- BITBLK128V32_16(ip, 1, op, parm);\
- BITBLK128V32_16(ip, 2, op, parm);\
- BITBLK128V32_16(ip, 3, op, parm);\
- BITBLK128V32_16(ip, 4, op, parm);\
- BITBLK128V32_16(ip, 5, op, parm);\
- BITBLK128V32_16(ip, 6, op, parm);\
- BITBLK128V32_16(ip, 7, op, parm);\
- BITBLK128V32_16(ip, 8, op, parm);\
- BITBLK128V32_16(ip, 9, op, parm);\
- BITBLK128V32_16(ip, 10, op, parm);\
- BITBLK128V32_16(ip, 11, op, parm);\
- BITBLK128V32_16(ip, 12, op, parm);\
- BITBLK128V32_16(ip, 13, op, parm);\
- BITBLK128V32_16(ip, 14, op, parm);\
- BITBLK128V32_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_17(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 1));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 3));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 5));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 7));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 9));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 11));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 13));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 15)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_17(ip, op, parm) {\
- BITBLK128V32_17(ip, 0, op, parm); IPPE(ip); OPPE(op += 17*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_18(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 2));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+11, iv), 6));\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+13, iv), 10));\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 14)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_18(ip, op, parm) {\
- BITBLK128V32_18(ip, 0, op, parm);\
- BITBLK128V32_18(ip, 1, op, parm); IPPE(ip); OPPE(op += 18*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_19(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 5));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 11));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 3));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 9));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 1));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 7));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 13)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_19(ip, op, parm) {\
- BITBLK128V32_19(ip, 0, op, parm); IPPE(ip); OPPE(op += 19*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_20(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 5, iv), 4));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 12)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_20(ip, op, parm) {\
- BITBLK128V32_20(ip, 0, op, parm);\
- BITBLK128V32_20(ip, 1, op, parm);\
- BITBLK128V32_20(ip, 2, op, parm);\
- BITBLK128V32_20(ip, 3, op, parm); IPPE(ip); OPPE(op += 20*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_21(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 5, iv), 9));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+11, iv), 7));\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 5));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+23, iv), 3));\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+29, iv), 1));\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 11)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_21(ip, op, parm) {\
- BITBLK128V32_21(ip, 0, op, parm); IPPE(ip); OPPE(op += 21*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_22(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 3, iv), 2));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 9, iv), 6));\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 10)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_22(ip, op, parm) {\
- BITBLK128V32_22(ip, 0, op, parm);\
- BITBLK128V32_22(ip, 1, op, parm); IPPE(ip); OPPE(op += 22*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_23(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 3, iv), 5));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 7, iv), 1));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+17, iv), 7));\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 3));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 9)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_23(ip, op, parm) {\
- BITBLK128V32_23(ip, 0, op, parm); IPPE(ip); OPPE(op += 23*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_24(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*4+ 1, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*4+ 2, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*4+ 3, iv), 8)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_24(ip, op, parm) {\
- BITBLK128V32_24(ip, 0, op, parm);\
- BITBLK128V32_24(ip, 1, op, parm);\
- BITBLK128V32_24(ip, 2, op, parm);\
- BITBLK128V32_24(ip, 3, op, parm);\
- BITBLK128V32_24(ip, 4, op, parm);\
- BITBLK128V32_24(ip, 5, op, parm);\
- BITBLK128V32_24(ip, 6, op, parm);\
- BITBLK128V32_24(ip, 7, op, parm); IPPE(ip); OPPE(op += 24*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_25(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 9, iv), 1));\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+13, iv), 5));\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+27, iv), 3));\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 7)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_25(ip, op, parm) {\
- BITBLK128V32_25(ip, 0, op, parm); IPPE(ip); OPPE(op += 25*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_26(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+ 5, iv), 2));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 6)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_26(ip, op, parm) {\
- BITBLK128V32_26(ip, 0, op, parm);\
- BITBLK128V32_26(ip, 1, op, parm); IPPE(ip); OPPE(op += 26*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_27(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+19, iv), 1));\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+25, iv), 3));\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 5)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_27(ip, op, parm) {\
- BITBLK128V32_27(ip, 0, op, parm); IPPE(ip); OPPE(op += 27*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_28(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*8+ 7, iv), 4)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_28(ip, op, parm) {\
- BITBLK128V32_28(ip, 0, op, parm);\
- BITBLK128V32_28(ip, 1, op, parm);\
- BITBLK128V32_28(ip, 2, op, parm);\
- BITBLK128V32_28(ip, 3, op, parm); IPPE(ip); OPPE(op += 28*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_29(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 27);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+21, iv), 1));\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 3)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_29(ip, op, parm) {\
- BITBLK128V32_29(ip, 0, op, parm); IPPE(ip); OPPE(op += 29*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_30(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*16+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+10, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*16+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+11, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*16+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+12, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*16+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+13, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*16+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*16+14, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*16+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*16+15, iv), 2)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_30(ip, op, parm) {\
- BITBLK128V32_30(ip, 0, op, parm);\
- BITBLK128V32_30(ip, 1, op, parm); IPPE(ip); OPPE(op += 30*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_31(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 31)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 29)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 27)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 25)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 23)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 9);\
- VI32(ip, i*32+10, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+11, iv), 21)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 11);\
- VI32(ip, i*32+12, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+13, iv), 19)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 13);\
- VI32(ip, i*32+14, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+15, iv), 17)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 15);\
- VI32(ip, i*32+16, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+17, iv), 15)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 17);\
- VI32(ip, i*32+18, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+18, iv), 14)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 18);\
- VI32(ip, i*32+19, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+19, iv), 13)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 19);\
- VI32(ip, i*32+20, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+21, iv), 11)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 21);\
- VI32(ip, i*32+22, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+22, iv), 10)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 22);\
- VI32(ip, i*32+23, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+23, iv), 9)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 23);\
- VI32(ip, i*32+24, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+25, iv), 7)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 25);\
- VI32(ip, i*32+26, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+26, iv), 6)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 26);\
- VI32(ip, i*32+27, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+27, iv), 5)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 27);\
- VI32(ip, i*32+28, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+28, iv), 4)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 28);\
- VI32(ip, i*32+29, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+29, iv), 3)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 29);\
- VI32(ip, i*32+30, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32(iv = IP32(ip, i*32+30, iv), 2)); _mm_storeu_si128(op++, ov); ov = mm_srli_epi32(iv, 30);\
- VI32(ip, i*32+31, iv, parm); ov = _mm_or_si128(ov, mm_slli_epi32( IP32(ip, i*32+31, iv), 1)); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_31(ip, op, parm) {\
- BITBLK128V32_31(ip, 0, op, parm); IPPE(ip); OPPE(op += 31*4/sizeof(op[0]));\
-}
-
-#define BITBLK128V32_32(ip, i, op, parm) { __m128i ov,iv;\
- VI32(ip, i*1+ 0, iv, parm); ov = IP32(ip, i*1+ 0, iv); _mm_storeu_si128((__m128i *)op++, ov);\
-}
-
-#define BITPACK128V32_32(ip, op, parm) {\
- BITBLK128V32_32(ip, 0, op, parm);\
- BITBLK128V32_32(ip, 1, op, parm);\
- BITBLK128V32_32(ip, 2, op, parm);\
- BITBLK128V32_32(ip, 3, op, parm);\
- BITBLK128V32_32(ip, 4, op, parm);\
- BITBLK128V32_32(ip, 5, op, parm);\
- BITBLK128V32_32(ip, 6, op, parm);\
- BITBLK128V32_32(ip, 7, op, parm);\
- BITBLK128V32_32(ip, 8, op, parm);\
- BITBLK128V32_32(ip, 9, op, parm);\
- BITBLK128V32_32(ip, 10, op, parm);\
- BITBLK128V32_32(ip, 11, op, parm);\
- BITBLK128V32_32(ip, 12, op, parm);\
- BITBLK128V32_32(ip, 13, op, parm);\
- BITBLK128V32_32(ip, 14, op, parm);\
- BITBLK128V32_32(ip, 15, op, parm);\
- BITBLK128V32_32(ip, 16, op, parm);\
- BITBLK128V32_32(ip, 17, op, parm);\
- BITBLK128V32_32(ip, 18, op, parm);\
- BITBLK128V32_32(ip, 19, op, parm);\
- BITBLK128V32_32(ip, 20, op, parm);\
- BITBLK128V32_32(ip, 21, op, parm);\
- BITBLK128V32_32(ip, 22, op, parm);\
- BITBLK128V32_32(ip, 23, op, parm);\
- BITBLK128V32_32(ip, 24, op, parm);\
- BITBLK128V32_32(ip, 25, op, parm);\
- BITBLK128V32_32(ip, 26, op, parm);\
- BITBLK128V32_32(ip, 27, op, parm);\
- BITBLK128V32_32(ip, 28, op, parm);\
- BITBLK128V32_32(ip, 29, op, parm);\
- BITBLK128V32_32(ip, 30, op, parm);\
- BITBLK128V32_32(ip, 31, op, parm); IPPE(ip); OPPE(op += 32*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_1(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 1));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 3));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 5));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 7));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 9));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 11));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 13));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 15));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 17));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 18));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 19));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 21));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 22));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 23));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 25));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 26));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 27));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 28));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 29));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 30));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 31)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_1(ip, op, parm) {\
- BITBLK256V32_1(ip, 0, op, parm); IPPE(ip); OPPE(op += 1*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_2(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 2));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 6));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 10));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 14));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 18));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 20));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 22));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 24));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 26));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 28));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 30)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_2(ip, op, parm) {\
- BITBLK256V32_2(ip, 0, op, parm);\
- BITBLK256V32_2(ip, 1, op, parm); IPPE(ip); OPPE(op += 2*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_3(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 3));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 9));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 15));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 18));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 21));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 27));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 1));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 7));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 13));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 19));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 22));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 25));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 28));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 5));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 11));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 14));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 17));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 23));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 26));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 29)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_3(ip, op, parm) {\
- BITBLK256V32_3(ip, 0, op, parm); IPPE(ip); OPPE(op += 3*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_4(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 1, iv), 4));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 3, iv), 12));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 5, iv), 20));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 6, iv), 24));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 28)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_4(ip, op, parm) {\
- BITBLK256V32_4(ip, 0, op, parm);\
- BITBLK256V32_4(ip, 1, op, parm);\
- BITBLK256V32_4(ip, 2, op, parm);\
- BITBLK256V32_4(ip, 3, op, parm); IPPE(ip); OPPE(op += 4*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_5(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 5));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 15));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 20));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 25));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 3));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 13));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 18));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 23));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 1));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 11));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 21));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 26));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 9));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 14));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 19));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 24));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 7));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 17));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 22));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 27)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_5(ip, op, parm) {\
- BITBLK256V32_5(ip, 0, op, parm); IPPE(ip); OPPE(op += 5*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_6(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 6));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 12));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 18));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 24));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 10));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 22));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 2));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 14));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 20));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 26)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_6(ip, op, parm) {\
- BITBLK256V32_6(ip, 0, op, parm);\
- BITBLK256V32_6(ip, 1, op, parm); IPPE(ip); OPPE(op += 6*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_7(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 7));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 14));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 21));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 3));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 10));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 17));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 24));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 13));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 20));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 9));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 23));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 5));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 19));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 1));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 15));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 22));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 11));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 18));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 25)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_7(ip, op, parm) {\
- BITBLK256V32_7(ip, 0, op, parm); IPPE(ip); OPPE(op += 7*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_8(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 1, iv), 8));\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 2, iv), 16));\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 3, iv), 24)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_8(ip, op, parm) {\
- BITBLK256V32_8(ip, 0, op, parm);\
- BITBLK256V32_8(ip, 1, op, parm);\
- BITBLK256V32_8(ip, 2, op, parm);\
- BITBLK256V32_8(ip, 3, op, parm);\
- BITBLK256V32_8(ip, 4, op, parm);\
- BITBLK256V32_8(ip, 5, op, parm);\
- BITBLK256V32_8(ip, 6, op, parm);\
- BITBLK256V32_8(ip, 7, op, parm); IPPE(ip); OPPE(op += 8*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_9(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 9));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 18));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 13));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 22));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 17));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 3));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 21));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 7));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 11));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 20));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 15));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 1));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 10));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 19));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 5));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 14));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 23)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_9(ip, op, parm) {\
- BITBLK256V32_9(ip, 0, op, parm); IPPE(ip); OPPE(op += 9*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_10(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 10));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 20));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 18));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 6));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 14));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 2));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 12));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 22)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_10(ip, op, parm) {\
- BITBLK256V32_10(ip, 0, op, parm);\
- BITBLK256V32_10(ip, 1, op, parm); IPPE(ip); OPPE(op += 10*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_11(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 11));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 1));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 13));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 3));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 14));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 15));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 5));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 6));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 17));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 7));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 18));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 19));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 9));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 20));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 10));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 21)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_11(ip, op, parm) {\
- BITBLK256V32_11(ip, 0, op, parm); IPPE(ip); OPPE(op += 11*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_12(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 1, iv), 12));\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 3, iv), 4));\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 4, iv), 16));\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 6, iv), 8));\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 20)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_12(ip, op, parm) {\
- BITBLK256V32_12(ip, 0, op, parm);\
- BITBLK256V32_12(ip, 1, op, parm);\
- BITBLK256V32_12(ip, 2, op, parm);\
- BITBLK256V32_12(ip, 3, op, parm); IPPE(ip); OPPE(op += 12*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_13(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 13));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 7));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 1));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 14));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 15));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 9));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 3));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 10));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 17));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 11));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 5));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 18));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 12));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 6));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 19)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_13(ip, op, parm) {\
- BITBLK256V32_13(ip, 0, op, parm); IPPE(ip); OPPE(op += 13*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_14(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 1, iv), 14));\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 10));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 6));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 7, iv), 2));\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 8, iv), 16));\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 12));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+14, iv), 4));\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 18)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_14(ip, op, parm) {\
- BITBLK256V32_14(ip, 0, op, parm);\
- BITBLK256V32_14(ip, 1, op, parm); IPPE(ip); OPPE(op += 14*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_15(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 1, iv), 15));\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 13));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 11));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 9));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 7));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 5));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 3));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+15, iv), 1));\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+16, iv), 16));\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 14));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 12));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 10));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 6));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+30, iv), 2));\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 17)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_15(ip, op, parm) {\
- BITBLK256V32_15(ip, 0, op, parm); IPPE(ip); OPPE(op += 15*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_16(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*2+ 0, iv, parm); ov = IP32(ip, i*2+ 0, iv);\
- VI32(ip, i*2+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*2+ 1, iv), 16)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_16(ip, op, parm) {\
- BITBLK256V32_16(ip, 0, op, parm);\
- BITBLK256V32_16(ip, 1, op, parm);\
- BITBLK256V32_16(ip, 2, op, parm);\
- BITBLK256V32_16(ip, 3, op, parm);\
- BITBLK256V32_16(ip, 4, op, parm);\
- BITBLK256V32_16(ip, 5, op, parm);\
- BITBLK256V32_16(ip, 6, op, parm);\
- BITBLK256V32_16(ip, 7, op, parm);\
- BITBLK256V32_16(ip, 8, op, parm);\
- BITBLK256V32_16(ip, 9, op, parm);\
- BITBLK256V32_16(ip, 10, op, parm);\
- BITBLK256V32_16(ip, 11, op, parm);\
- BITBLK256V32_16(ip, 12, op, parm);\
- BITBLK256V32_16(ip, 13, op, parm);\
- BITBLK256V32_16(ip, 14, op, parm);\
- BITBLK256V32_16(ip, 15, op, parm); IPPE(ip); OPPE(op += 16*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_17(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 2));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 6));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 10));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 12));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 14));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 1));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 3));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 5));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 7));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 9));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 11));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 13));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 15)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_17(ip, op, parm) {\
- BITBLK256V32_17(ip, 0, op, parm); IPPE(ip); OPPE(op += 17*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_18(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 2, iv), 4));\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 4, iv), 8));\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 12));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 2));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+11, iv), 6));\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+13, iv), 10));\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 14)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_18(ip, op, parm) {\
- BITBLK256V32_18(ip, 0, op, parm);\
- BITBLK256V32_18(ip, 1, op, parm); IPPE(ip); OPPE(op += 18*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_19(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 6));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 12));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 5));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 11));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 10));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 3));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 9));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 2));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 1));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 7));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 13)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_19(ip, op, parm) {\
- BITBLK256V32_19(ip, 0, op, parm); IPPE(ip); OPPE(op += 19*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_20(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 2, iv), 8));\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 5, iv), 4));\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 12)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_20(ip, op, parm) {\
- BITBLK256V32_20(ip, 0, op, parm);\
- BITBLK256V32_20(ip, 1, op, parm);\
- BITBLK256V32_20(ip, 2, op, parm);\
- BITBLK256V32_20(ip, 3, op, parm); IPPE(ip); OPPE(op += 20*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_21(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 2, iv), 10));\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 5, iv), 9));\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 8, iv), 8));\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+11, iv), 7));\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 6));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 5));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+20, iv), 4));\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+23, iv), 3));\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+26, iv), 2));\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+29, iv), 1));\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 11)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_21(ip, op, parm) {\
- BITBLK256V32_21(ip, 0, op, parm); IPPE(ip); OPPE(op += 21*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_22(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 3, iv), 2));\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 6, iv), 4));\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 9, iv), 6));\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+12, iv), 8));\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 10)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_22(ip, op, parm) {\
- BITBLK256V32_22(ip, 0, op, parm);\
- BITBLK256V32_22(ip, 1, op, parm); IPPE(ip); OPPE(op += 22*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_23(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 3, iv), 5));\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 7, iv), 1));\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 6));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+14, iv), 2));\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+17, iv), 7));\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 3));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+24, iv), 8));\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+28, iv), 4));\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 9)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_23(ip, op, parm) {\
- BITBLK256V32_23(ip, 0, op, parm); IPPE(ip); OPPE(op += 23*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_24(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*4+ 0, iv, parm); ov = IP32(ip, i*4+ 0, iv);\
- VI32(ip, i*4+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*4+ 1, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*4+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*4+ 2, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*4+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*4+ 3, iv), 8)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_24(ip, op, parm) {\
- BITBLK256V32_24(ip, 0, op, parm);\
- BITBLK256V32_24(ip, 1, op, parm);\
- BITBLK256V32_24(ip, 2, op, parm);\
- BITBLK256V32_24(ip, 3, op, parm);\
- BITBLK256V32_24(ip, 4, op, parm);\
- BITBLK256V32_24(ip, 5, op, parm);\
- BITBLK256V32_24(ip, 6, op, parm);\
- BITBLK256V32_24(ip, 7, op, parm); IPPE(ip); OPPE(op += 24*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_25(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 4, iv), 4));\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 9, iv), 1));\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+13, iv), 5));\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+18, iv), 2));\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+22, iv), 6));\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+27, iv), 3));\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 7)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_25(ip, op, parm) {\
- BITBLK256V32_25(ip, 0, op, parm); IPPE(ip); OPPE(op += 25*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_26(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+ 5, iv), 2));\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+10, iv), 4));\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 6)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_26(ip, op, parm) {\
- BITBLK256V32_26(ip, 0, op, parm);\
- BITBLK256V32_26(ip, 1, op, parm); IPPE(ip); OPPE(op += 26*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_27(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+ 6, iv), 2));\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+12, iv), 4));\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+19, iv), 1));\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+25, iv), 3));\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 5)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_27(ip, op, parm) {\
- BITBLK256V32_27(ip, 0, op, parm); IPPE(ip); OPPE(op += 27*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_28(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*8+ 0, iv, parm); ov = IP32(ip, i*8+ 0, iv);\
- VI32(ip, i*8+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 1, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*8+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 2, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*8+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 3, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*8+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 4, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*8+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 5, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*8+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*8+ 6, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*8+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*8+ 7, iv), 4)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_28(ip, op, parm) {\
- BITBLK256V32_28(ip, 0, op, parm);\
- BITBLK256V32_28(ip, 1, op, parm);\
- BITBLK256V32_28(ip, 2, op, parm);\
- BITBLK256V32_28(ip, 3, op, parm); IPPE(ip); OPPE(op += 28*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_29(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 5)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 27);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+10, iv), 2));\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+21, iv), 1));\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 3)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_29(ip, op, parm) {\
- BITBLK256V32_29(ip, 0, op, parm); IPPE(ip); OPPE(op += 29*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_30(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*16+ 0, iv, parm); ov = IP32(ip, i*16+ 0, iv);\
- VI32(ip, i*16+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 1, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*16+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 2, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*16+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 3, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*16+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 4, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*16+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 5, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*16+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 6, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*16+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 7, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*16+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 8, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*16+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+ 9, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*16+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+10, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*16+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+11, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*16+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+12, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*16+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+13, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*16+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*16+14, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*16+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*16+15, iv), 2)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_30(ip, op, parm) {\
- BITBLK256V32_30(ip, 0, op, parm);\
- BITBLK256V32_30(ip, 1, op, parm); IPPE(ip); OPPE(op += 30*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_31(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*32+ 0, iv, parm); ov = IP32(ip, i*32+ 0, iv);\
- VI32(ip, i*32+ 1, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 1, iv), 31)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 1);\
- VI32(ip, i*32+ 2, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 2, iv), 30)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 2);\
- VI32(ip, i*32+ 3, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 3, iv), 29)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 3);\
- VI32(ip, i*32+ 4, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 4, iv), 28)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 4);\
- VI32(ip, i*32+ 5, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 5, iv), 27)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 5);\
- VI32(ip, i*32+ 6, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 6, iv), 26)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 6);\
- VI32(ip, i*32+ 7, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 7, iv), 25)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 7);\
- VI32(ip, i*32+ 8, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 8, iv), 24)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 8);\
- VI32(ip, i*32+ 9, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+ 9, iv), 23)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 9);\
- VI32(ip, i*32+10, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+10, iv), 22)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 10);\
- VI32(ip, i*32+11, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+11, iv), 21)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 11);\
- VI32(ip, i*32+12, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+12, iv), 20)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 12);\
- VI32(ip, i*32+13, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+13, iv), 19)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 13);\
- VI32(ip, i*32+14, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+14, iv), 18)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 14);\
- VI32(ip, i*32+15, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+15, iv), 17)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 15);\
- VI32(ip, i*32+16, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+16, iv), 16)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 16);\
- VI32(ip, i*32+17, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+17, iv), 15)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 17);\
- VI32(ip, i*32+18, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+18, iv), 14)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 18);\
- VI32(ip, i*32+19, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+19, iv), 13)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 19);\
- VI32(ip, i*32+20, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+20, iv), 12)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 20);\
- VI32(ip, i*32+21, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+21, iv), 11)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 21);\
- VI32(ip, i*32+22, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+22, iv), 10)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 22);\
- VI32(ip, i*32+23, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+23, iv), 9)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 23);\
- VI32(ip, i*32+24, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+24, iv), 8)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 24);\
- VI32(ip, i*32+25, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+25, iv), 7)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 25);\
- VI32(ip, i*32+26, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+26, iv), 6)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 26);\
- VI32(ip, i*32+27, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+27, iv), 5)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 27);\
- VI32(ip, i*32+28, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+28, iv), 4)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 28);\
- VI32(ip, i*32+29, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+29, iv), 3)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 29);\
- VI32(ip, i*32+30, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32(iv = IP32(ip, i*32+30, iv), 2)); _mm256_storeu_si256(op++, ov); ov = _mm256_srli_epi32(iv, 30);\
- VI32(ip, i*32+31, iv, parm); ov = _mm256_or_si256(ov, _mm256_slli_epi32( IP32(ip, i*32+31, iv), 1)); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_31(ip, op, parm) {\
- BITBLK256V32_31(ip, 0, op, parm); IPPE(ip); OPPE(op += 31*4/sizeof(op[0]));\
-}
-
-#define BITBLK256V32_32(ip, i, op, parm) { __m256i ov,iv;\
- VI32(ip, i*1+ 0, iv, parm); ov = IP32(ip, i*1+ 0, iv); _mm256_storeu_si256((__m128i *)op++, ov);\
-}
-
-#define BITPACK256V32_32(ip, op, parm) {\
- BITBLK256V32_32(ip, 0, op, parm);\
- BITBLK256V32_32(ip, 1, op, parm);\
- BITBLK256V32_32(ip, 2, op, parm);\
- BITBLK256V32_32(ip, 3, op, parm);\
- BITBLK256V32_32(ip, 4, op, parm);\
- BITBLK256V32_32(ip, 5, op, parm);\
- BITBLK256V32_32(ip, 6, op, parm);\
- BITBLK256V32_32(ip, 7, op, parm);\
- BITBLK256V32_32(ip, 8, op, parm);\
- BITBLK256V32_32(ip, 9, op, parm);\
- BITBLK256V32_32(ip, 10, op, parm);\
- BITBLK256V32_32(ip, 11, op, parm);\
- BITBLK256V32_32(ip, 12, op, parm);\
- BITBLK256V32_32(ip, 13, op, parm);\
- BITBLK256V32_32(ip, 14, op, parm);\
- BITBLK256V32_32(ip, 15, op, parm);\
- BITBLK256V32_32(ip, 16, op, parm);\
- BITBLK256V32_32(ip, 17, op, parm);\
- BITBLK256V32_32(ip, 18, op, parm);\
- BITBLK256V32_32(ip, 19, op, parm);\
- BITBLK256V32_32(ip, 20, op, parm);\
- BITBLK256V32_32(ip, 21, op, parm);\
- BITBLK256V32_32(ip, 22, op, parm);\
- BITBLK256V32_32(ip, 23, op, parm);\
- BITBLK256V32_32(ip, 24, op, parm);\
- BITBLK256V32_32(ip, 25, op, parm);\
- BITBLK256V32_32(ip, 26, op, parm);\
- BITBLK256V32_32(ip, 27, op, parm);\
- BITBLK256V32_32(ip, 28, op, parm);\
- BITBLK256V32_32(ip, 29, op, parm);\
- BITBLK256V32_32(ip, 30, op, parm);\
- BITBLK256V32_32(ip, 31, op, parm); IPPE(ip); OPPE(op += 32*4/sizeof(op[0]));\
-}
-
-//#include "bitpack_0.h"
-#define BITPACK128V16(_ip_, _nbits_, _op_, __parm) { __m128i *_ip=(__m128i *)_ip_,*_op=(__m128i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK128V16_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK128V16_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK128V16_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK128V16_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK128V16_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK128V16_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK128V16_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK128V16_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK128V16_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK128V16_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK128V16_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK128V16_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK128V16_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK128V16_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK128V16_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK128V16_16(_ip, _op, __parm); } break;\
- }\
-}
-
-#define BITPACK128V32(_ip_, _nbits_, _op_, __parm) { __m128i *_ip=(__m128i *)_ip_,*_op=(__m128i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK128V32_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK128V32_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK128V32_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK128V32_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK128V32_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK128V32_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK128V32_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK128V32_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK128V32_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK128V32_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK128V32_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK128V32_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK128V32_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK128V32_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK128V32_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK128V32_16(_ip, _op, __parm); } break;\
- case 17:{ BITPACK128V32_17(_ip, _op, __parm); } break;\
- case 18:{ BITPACK128V32_18(_ip, _op, __parm); } break;\
- case 19:{ BITPACK128V32_19(_ip, _op, __parm); } break;\
- case 20:{ BITPACK128V32_20(_ip, _op, __parm); } break;\
- case 21:{ BITPACK128V32_21(_ip, _op, __parm); } break;\
- case 22:{ BITPACK128V32_22(_ip, _op, __parm); } break;\
- case 23:{ BITPACK128V32_23(_ip, _op, __parm); } break;\
- case 24:{ BITPACK128V32_24(_ip, _op, __parm); } break;\
- case 25:{ BITPACK128V32_25(_ip, _op, __parm); } break;\
- case 26:{ BITPACK128V32_26(_ip, _op, __parm); } break;\
- case 27:{ BITPACK128V32_27(_ip, _op, __parm); } break;\
- case 28:{ BITPACK128V32_28(_ip, _op, __parm); } break;\
- case 29:{ BITPACK128V32_29(_ip, _op, __parm); } break;\
- case 30:{ BITPACK128V32_30(_ip, _op, __parm); } break;\
- case 31:{ BITPACK128V32_31(_ip, _op, __parm); } break;\
- case 32:{ BITPACK128V32_32(_ip, _op, __parm); } break;\
- }\
-}
-
-#define BITPACK256V32(_ip_, _nbits_, _op_, __parm) { __m256i *_ip=(__m256i *)_ip_,*_op=(__m256i *)_op_;\
- switch(_nbits_) {\
- case 0: break;\
- case 1:{ BITPACK256V32_1( _ip, _op, __parm); } break;\
- case 2:{ BITPACK256V32_2( _ip, _op, __parm); } break;\
- case 3:{ BITPACK256V32_3( _ip, _op, __parm); } break;\
- case 4:{ BITPACK256V32_4( _ip, _op, __parm); } break;\
- case 5:{ BITPACK256V32_5( _ip, _op, __parm); } break;\
- case 6:{ BITPACK256V32_6( _ip, _op, __parm); } break;\
- case 7:{ BITPACK256V32_7( _ip, _op, __parm); } break;\
- case 8:{ BITPACK256V32_8( _ip, _op, __parm); } break;\
- case 9:{ BITPACK256V32_9( _ip, _op, __parm); } break;\
- case 10:{ BITPACK256V32_10(_ip, _op, __parm); } break;\
- case 11:{ BITPACK256V32_11(_ip, _op, __parm); } break;\
- case 12:{ BITPACK256V32_12(_ip, _op, __parm); } break;\
- case 13:{ BITPACK256V32_13(_ip, _op, __parm); } break;\
- case 14:{ BITPACK256V32_14(_ip, _op, __parm); } break;\
- case 15:{ BITPACK256V32_15(_ip, _op, __parm); } break;\
- case 16:{ BITPACK256V32_16(_ip, _op, __parm); } break;\
- case 17:{ BITPACK256V32_17(_ip, _op, __parm); } break;\
- case 18:{ BITPACK256V32_18(_ip, _op, __parm); } break;\
- case 19:{ BITPACK256V32_19(_ip, _op, __parm); } break;\
- case 20:{ BITPACK256V32_20(_ip, _op, __parm); } break;\
- case 21:{ BITPACK256V32_21(_ip, _op, __parm); } break;\
- case 22:{ BITPACK256V32_22(_ip, _op, __parm); } break;\
- case 23:{ BITPACK256V32_23(_ip, _op, __parm); } break;\
- case 24:{ BITPACK256V32_24(_ip, _op, __parm); } break;\
- case 25:{ BITPACK256V32_25(_ip, _op, __parm); } break;\
- case 26:{ BITPACK256V32_26(_ip, _op, __parm); } break;\
- case 27:{ BITPACK256V32_27(_ip, _op, __parm); } break;\
- case 28:{ BITPACK256V32_28(_ip, _op, __parm); } break;\
- case 29:{ BITPACK256V32_29(_ip, _op, __parm); } break;\
- case 30:{ BITPACK256V32_30(_ip, _op, __parm); } break;\
- case 31:{ BITPACK256V32_31(_ip, _op, __parm); } break;\
- case 32:{ BITPACK256V32_32(_ip, _op, __parm); } break;\
- }\
-}
-
diff --git a/src/ext/for/bitunpack.c b/src/ext/for/bitunpack.c
deleted file mode 100644
index 1dd78003ada..00000000000
--- a/src/ext/for/bitunpack.c
+++ /dev/null
@@ -1,1231 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2019
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// "Integer Compression" Bit Packing
-#define BITUTIL_IN
-#define VINT_IN
-#include "conf.h"
-#include "bitutil.h"
-#include "bitpack.h"
-#include "vint.h"
-
-#define PAD8(_x_) (((_x_)+7)/8)
-
-#pragma warning( disable : 4005)
-#pragma warning( disable : 4090)
-#pragma warning( disable : 4068)
-
-#pragma GCC push_options
-#pragma GCC optimize ("align-functions=16")
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunsequenced"
-
-#if !defined(SSE2_ON) && !defined(AVX2_ON) //----------------------------------- Plain -------------------------------------------------------------------------------------------
-typedef unsigned char *(*BITUNPACK_F8)( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D8)( const unsigned char *__restrict in, unsigned n, uint8_t *__restrict out, uint8_t start);
-typedef unsigned char *(*BITUNPACK_F16)(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D16)(const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start);
-typedef unsigned char *(*BITUNPACK_F32)(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D32)(const unsigned char *__restrict in, unsigned n, uint32_t *__restrict out, uint32_t start);
-typedef unsigned char *(*BITUNPACK_F64)(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out);
-typedef unsigned char *(*BITUNPACK_D64)(const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start);
-
- #if 0 //????
-#define OP(_op_, _x_) *_op_++
-#define OPX(_op_)
- #else
-#define OP(_op_, _x_) _op_[_x_]
-#define OPX(_op_) _op_ += 32
- #endif
-
-#define OPI(_op_,_nb_,_parm_) OPX(_op_)
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = _w_
-#define _BITUNPACK_ bitunpack
-#include "bitunpack_.h"
-
-#define DELTA
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += (_w_))
-#define _BITUNPACK_ bitdunpack // delta + 0
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += TEMPLATE2(zigzagdec, USIZE)(_w_))
-#define _BITUNPACK_ bitzunpack // zigzag
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ + (_w_))
-#define _BITUNPACK_ bitfunpack // for
-#include "bitunpack_.h"
-
-#define OPI(_op_,_nb_,_parm_) OPX(_op_); _parm_ += 32
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = (_parm_ += (_w_)) + (_x_+1)
-#define _BITUNPACK_ bitd1unpack // delta + 1
-#include "bitunpack_.h"
-
-#define OUT( _op_, _x_, _w_, _nb_,_parm_) OP(_op_,_x_) = _parm_ + (_w_)+(_x_+1)
-#define _BITUNPACK_ bitf1unpack // for + 1
-#include "bitunpack_.h"
-#undef OPI
-
-#define BITNUNPACK(in, n, out, _csize_, _usize_) {\
- unsigned char *ip = in;\
- for(op = out,out+=n; op < out;) { unsigned oplen = out - op,b; if(oplen > _csize_) oplen = _csize_; PREFETCH(ip+512,0);\
- b = *ip++; ip = TEMPLATE2(bitunpacka, _usize_)[b](ip, oplen, op);\
- op += oplen;\
- } \
- return ip - in;\
-}
-
-#define BITNDUNPACK(in, n, out, _csize_, _usize_, _bitunpacka_) { if(!n) return 0;\
- unsigned char *ip = in;\
- TEMPLATE2(vbxget, _usize_)(ip, start);\
- for(*out++ = start,--n,op = out; op != out+(n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpacka_, _usize_)[b](ip, _csize_, op, start); op += _csize_; start = op[-1];\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(_bitunpacka_, _usize_)[b](ip, n, op, start); }\
- return ip - in;\
-}
-
-size_t bitnunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op; BITNUNPACK(in, n, out, 128, 8); }
-size_t bitnunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op; BITNUNPACK(in, n, out, 128, 16); }
-size_t bitnunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; BITNUNPACK(in, n, out, 128, 32); }
-size_t bitnunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op; BITNUNPACK(in, n, out, 128, 64); }
-
-size_t bitndunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitdunpacka); }
-size_t bitndunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitdunpacka); }
-size_t bitndunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitdunpacka); }
-size_t bitndunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitdunpacka); }
-
-size_t bitnd1unpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitd1unpacka); }
-size_t bitnd1unpack16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitd1unpacka); }
-size_t bitnd1unpack32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitd1unpacka); }
-size_t bitnd1unpack64(unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitd1unpacka); }
-
-size_t bitnzunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitzunpacka); }
-size_t bitnzunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitzunpacka); }
-size_t bitnzunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitzunpacka); }
-size_t bitnzunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitzunpacka); }
-
-size_t bitnfunpack8( unsigned char *__restrict in, size_t n, uint8_t *__restrict out) { uint8_t *op,start; BITNDUNPACK(in, n, out, 128, 8, bitfunpacka); }
-size_t bitnfunpack16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; BITNDUNPACK(in, n, out, 128, 16, bitfunpacka); }
-size_t bitnfunpack32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; BITNDUNPACK(in, n, out, 128, 32, bitfunpacka); }
-size_t bitnfunpack64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op,start; BITNDUNPACK(in, n, out, 128, 64, bitfunpacka); }
-
-#else //-------------------------------------------- SSE/AVX2 ---------------------------------------------------------------------------------------
-
-#define _BITNUNPACKV(in, n, out, _csize_, _usize_, _bitunpackv_) {\
- unsigned char *ip = in;\
- for(op = out; op != out+(n&~(_csize_-1)); op += _csize_) { PREFETCH(in+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpackv_, _usize_)(ip, _csize_, op,b);\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(bitunpack, _usize_)(ip, n, op,b); }\
- return ip - in;\
-}
-
-#define _BITNDUNPACKV(in, n, out, _csize_, _usize_, _bitunpackv_, _bitunpack_) { if(!n) return 0;\
- unsigned char *ip = in;\
- TEMPLATE2(vbxget, _usize_)(ip, start); \
- *out++ = start;\
- for(--n,op = out; op != out+(n&~(_csize_-1)); ) { PREFETCH(ip+512,0);\
- unsigned b = *ip++; ip = TEMPLATE2(_bitunpackv_, _usize_)(ip, _csize_, op, start,b); op += _csize_; start = op[-1];\
- } if(n&=(_csize_-1)) { unsigned b = *ip++; ip = TEMPLATE2(_bitunpack_, _usize_)(ip, n, op, start,b); }\
- return ip - in;\
-}
- #ifdef __AVX2__ //-------------------------------- AVX2 ----------------------------------------------------------------------------
-#include <immintrin.h>
-
- #ifdef __AVX512F__
-#define mm256_maskz_expand_epi32(_m_,_v_) _mm256_maskz_expand_epi32(_m_,_v_)
-#define mm256_maskz_loadu_epi32( _m_,_v_) _mm256_maskz_loadu_epi32( _m_,_v_)
- #else
-#if !(defined(_M_X64) || defined(__amd64__)) && (defined(__i386__) || defined(_M_IX86)) && !defined(__clang__)
-static inline __m128i _mm_cvtsi64_si128(__int64 a) { return _mm_loadl_epi64((__m128i*)&a); }
- #endif
-static ALIGNED(unsigned char, permv[256][8], 32) = {
-0,0,0,0,0,0,0,0,
-0,1,1,1,1,1,1,1,
-1,0,1,1,1,1,1,1,
-0,1,2,2,2,2,2,2,
-1,1,0,1,1,1,1,1,
-0,2,1,2,2,2,2,2,
-2,0,1,2,2,2,2,2,
-0,1,2,3,3,3,3,3,
-1,1,1,0,1,1,1,1,
-0,2,2,1,2,2,2,2,
-2,0,2,1,2,2,2,2,
-0,1,3,2,3,3,3,3,
-2,2,0,1,2,2,2,2,
-0,3,1,2,3,3,3,3,
-3,0,1,2,3,3,3,3,
-0,1,2,3,4,4,4,4,
-1,1,1,1,0,1,1,1,
-0,2,2,2,1,2,2,2,
-2,0,2,2,1,2,2,2,
-0,1,3,3,2,3,3,3,
-2,2,0,2,1,2,2,2,
-0,3,1,3,2,3,3,3,
-3,0,1,3,2,3,3,3,
-0,1,2,4,3,4,4,4,
-2,2,2,0,1,2,2,2,
-0,3,3,1,2,3,3,3,
-3,0,3,1,2,3,3,3,
-0,1,4,2,3,4,4,4,
-3,3,0,1,2,3,3,3,
-0,4,1,2,3,4,4,4,
-4,0,1,2,3,4,4,4,
-0,1,2,3,4,5,5,5,
-1,1,1,1,1,0,1,1,
-0,2,2,2,2,1,2,2,
-2,0,2,2,2,1,2,2,
-0,1,3,3,3,2,3,3,
-2,2,0,2,2,1,2,2,
-0,3,1,3,3,2,3,3,
-3,0,1,3,3,2,3,3,
-0,1,2,4,4,3,4,4,
-2,2,2,0,2,1,2,2,
-0,3,3,1,3,2,3,3,
-3,0,3,1,3,2,3,3,
-0,1,4,2,4,3,4,4,
-3,3,0,1,3,2,3,3,
-0,4,1,2,4,3,4,4,
-4,0,1,2,4,3,4,4,
-0,1,2,3,5,4,5,5,
-2,2,2,2,0,1,2,2,
-0,3,3,3,1,2,3,3,
-3,0,3,3,1,2,3,3,
-0,1,4,4,2,3,4,4,
-3,3,0,3,1,2,3,3,
-0,4,1,4,2,3,4,4,
-4,0,1,4,2,3,4,4,
-0,1,2,5,3,4,5,5,
-3,3,3,0,1,2,3,3,
-0,4,4,1,2,3,4,4,
-4,0,4,1,2,3,4,4,
-0,1,5,2,3,4,5,5,
-4,4,0,1,2,3,4,4,
-0,5,1,2,3,4,5,5,
-5,0,1,2,3,4,5,5,
-0,1,2,3,4,5,6,6,
-1,1,1,1,1,1,0,1,
-0,2,2,2,2,2,1,2,
-2,0,2,2,2,2,1,2,
-0,1,3,3,3,3,2,3,
-2,2,0,2,2,2,1,2,
-0,3,1,3,3,3,2,3,
-3,0,1,3,3,3,2,3,
-0,1,2,4,4,4,3,4,
-2,2,2,0,2,2,1,2,
-0,3,3,1,3,3,2,3,
-3,0,3,1,3,3,2,3,
-0,1,4,2,4,4,3,4,
-3,3,0,1,3,3,2,3,
-0,4,1,2,4,4,3,4,
-4,0,1,2,4,4,3,4,
-0,1,2,3,5,5,4,5,
-2,2,2,2,0,2,1,2,
-0,3,3,3,1,3,2,3,
-3,0,3,3,1,3,2,3,
-0,1,4,4,2,4,3,4,
-3,3,0,3,1,3,2,3,
-0,4,1,4,2,4,3,4,
-4,0,1,4,2,4,3,4,
-0,1,2,5,3,5,4,5,
-3,3,3,0,1,3,2,3,
-0,4,4,1,2,4,3,4,
-4,0,4,1,2,4,3,4,
-0,1,5,2,3,5,4,5,
-4,4,0,1,2,4,3,4,
-0,5,1,2,3,5,4,5,
-5,0,1,2,3,5,4,5,
-0,1,2,3,4,6,5,6,
-2,2,2,2,2,0,1,2,
-0,3,3,3,3,1,2,3,
-3,0,3,3,3,1,2,3,
-0,1,4,4,4,2,3,4,
-3,3,0,3,3,1,2,3,
-0,4,1,4,4,2,3,4,
-4,0,1,4,4,2,3,4,
-0,1,2,5,5,3,4,5,
-3,3,3,0,3,1,2,3,
-0,4,4,1,4,2,3,4,
-4,0,4,1,4,2,3,4,
-0,1,5,2,5,3,4,5,
-4,4,0,1,4,2,3,4,
-0,5,1,2,5,3,4,5,
-5,0,1,2,5,3,4,5,
-0,1,2,3,6,4,5,6,
-3,3,3,3,0,1,2,3,
-0,4,4,4,1,2,3,4,
-4,0,4,4,1,2,3,4,
-0,1,5,5,2,3,4,5,
-4,4,0,4,1,2,3,4,
-0,5,1,5,2,3,4,5,
-5,0,1,5,2,3,4,5,
-0,1,2,6,3,4,5,6,
-4,4,4,0,1,2,3,4,
-0,5,5,1,2,3,4,5,
-5,0,5,1,2,3,4,5,
-0,1,6,2,3,4,5,6,
-5,5,0,1,2,3,4,5,
-0,6,1,2,3,4,5,6,
-6,0,1,2,3,4,5,6,
-0,1,2,3,4,5,6,7,
-1,1,1,1,1,1,1,0,
-0,2,2,2,2,2,2,1,
-2,0,2,2,2,2,2,1,
-0,1,3,3,3,3,3,2,
-2,2,0,2,2,2,2,1,
-0,3,1,3,3,3,3,2,
-3,0,1,3,3,3,3,2,
-0,1,2,4,4,4,4,3,
-2,2,2,0,2,2,2,1,
-0,3,3,1,3,3,3,2,
-3,0,3,1,3,3,3,2,
-0,1,4,2,4,4,4,3,
-3,3,0,1,3,3,3,2,
-0,4,1,2,4,4,4,3,
-4,0,1,2,4,4,4,3,
-0,1,2,3,5,5,5,4,
-2,2,2,2,0,2,2,1,
-0,3,3,3,1,3,3,2,
-3,0,3,3,1,3,3,2,
-0,1,4,4,2,4,4,3,
-3,3,0,3,1,3,3,2,
-0,4,1,4,2,4,4,3,
-4,0,1,4,2,4,4,3,
-0,1,2,5,3,5,5,4,
-3,3,3,0,1,3,3,2,
-0,4,4,1,2,4,4,3,
-4,0,4,1,2,4,4,3,
-0,1,5,2,3,5,5,4,
-4,4,0,1,2,4,4,3,
-0,5,1,2,3,5,5,4,
-5,0,1,2,3,5,5,4,
-0,1,2,3,4,6,6,5,
-2,2,2,2,2,0,2,1,
-0,3,3,3,3,1,3,2,
-3,0,3,3,3,1,3,2,
-0,1,4,4,4,2,4,3,
-3,3,0,3,3,1,3,2,
-0,4,1,4,4,2,4,3,
-4,0,1,4,4,2,4,3,
-0,1,2,5,5,3,5,4,
-3,3,3,0,3,1,3,2,
-0,4,4,1,4,2,4,3,
-4,0,4,1,4,2,4,3,
-0,1,5,2,5,3,5,4,
-4,4,0,1,4,2,4,3,
-0,5,1,2,5,3,5,4,
-5,0,1,2,5,3,5,4,
-0,1,2,3,6,4,6,5,
-3,3,3,3,0,1,3,2,
-0,4,4,4,1,2,4,3,
-4,0,4,4,1,2,4,3,
-0,1,5,5,2,3,5,4,
-4,4,0,4,1,2,4,3,
-0,5,1,5,2,3,5,4,
-5,0,1,5,2,3,5,4,
-0,1,2,6,3,4,6,5,
-4,4,4,0,1,2,4,3,
-0,5,5,1,2,3,5,4,
-5,0,5,1,2,3,5,4,
-0,1,6,2,3,4,6,5,
-5,5,0,1,2,3,5,4,
-0,6,1,2,3,4,6,5,
-6,0,1,2,3,4,6,5,
-0,1,2,3,4,5,7,6,
-2,2,2,2,2,2,0,1,
-0,3,3,3,3,3,1,2,
-3,0,3,3,3,3,1,2,
-0,1,4,4,4,4,2,3,
-3,3,0,3,3,3,1,2,
-0,4,1,4,4,4,2,3,
-4,0,1,4,4,4,2,3,
-0,1,2,5,5,5,3,4,
-3,3,3,0,3,3,1,2,
-0,4,4,1,4,4,2,3,
-4,0,4,1,4,4,2,3,
-0,1,5,2,5,5,3,4,
-4,4,0,1,4,4,2,3,
-0,5,1,2,5,5,3,4,
-5,0,1,2,5,5,3,4,
-0,1,2,3,6,6,4,5,
-3,3,3,3,0,3,1,2,
-0,4,4,4,1,4,2,3,
-4,0,4,4,1,4,2,3,
-0,1,5,5,2,5,3,4,
-4,4,0,4,1,4,2,3,
-0,5,1,5,2,5,3,4,
-5,0,1,5,2,5,3,4,
-0,1,2,6,3,6,4,5,
-4,4,4,0,1,4,2,3,
-0,5,5,1,2,5,3,4,
-5,0,5,1,2,5,3,4,
-0,1,6,2,3,6,4,5,
-5,5,0,1,2,5,3,4,
-0,6,1,2,3,6,4,5,
-6,0,1,2,3,6,4,5,
-0,1,2,3,4,7,5,6,
-3,3,3,3,3,0,1,2,
-0,4,4,4,4,1,2,3,
-4,0,4,4,4,1,2,3,
-0,1,5,5,5,2,3,4,
-4,4,0,4,4,1,2,3,
-0,5,1,5,5,2,3,4,
-5,0,1,5,5,2,3,4,
-0,1,2,6,6,3,4,5,
-4,4,4,0,4,1,2,3,
-0,5,5,1,5,2,3,4,
-5,0,5,1,5,2,3,4,
-0,1,6,2,6,3,4,5,
-5,5,0,1,5,2,3,4,
-0,6,1,2,6,3,4,5,
-6,0,1,2,6,3,4,5,
-0,1,2,3,7,4,5,6,
-4,4,4,4,0,1,2,3,
-0,5,5,5,1,2,3,4,
-5,0,5,5,1,2,3,4,
-0,1,6,6,2,3,4,5,
-5,5,0,5,1,2,3,4,
-0,6,1,6,2,3,4,5,
-6,0,1,6,2,3,4,5,
-0,1,2,7,3,4,5,6,
-5,5,5,0,1,2,3,4,
-0,6,6,1,2,3,4,5,
-6,0,6,1,2,3,4,5,
-0,1,7,2,3,4,5,6,
-6,6,0,1,2,3,4,5,
-0,7,1,2,3,4,5,6,
-7,0,1,2,3,4,5,6,
-0,1,2,3,4,5,6,7
-};
-#define u2vmask(_m_,_tv_) _mm256_sllv_epi32(_mm256_set1_epi8(_m_), _tv_)
-#define mm256_maskz_expand_epi32(_m_, _v_) _mm256_permutevar8x32_epi32(_v_, _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(ctou64(permv[_m_]))) )
-#define mm256_maskz_loadu_epi32(_m_,_v_) _mm256_blendv_epi8(zv, mm256_maskz_expand_epi32(xm, _mm256_loadu_si256((__m256i*)pex)), u2vmask(xm,tv)) // emulate AVX512 _mm256_maskz_loadu_epi32 on AVX2
- #endif
-
-//-----------------------------------------------------------------------------
-#define VO32( _op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, ov)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_)
-#include "bitunpack_.h"
-
-#define BITUNBLK256V32_0(ip, _i_, _op_, _nb_,_parm_) {__m256i ov;\
- VOZ32(_op_, 0, ov, _nb_,_parm_);\
- VOZ32(_op_, 1, ov, _nb_,_parm_);\
- VOZ32(_op_, 2, ov, _nb_,_parm_);\
- VOZ32(_op_, 3, ov, _nb_,_parm_);\
- VOZ32(_op_, 4, ov, _nb_,_parm_);\
- VOZ32(_op_, 5, ov, _nb_,_parm_);\
- VOZ32(_op_, 6, ov, _nb_,_parm_);\
- VOZ32(_op_, 7, ov, _nb_,_parm_);\
- VOZ32(_op_, 8, ov, _nb_,_parm_);\
- VOZ32(_op_, 9, ov, _nb_,_parm_);\
- VOZ32(_op_, 10, ov, _nb_,_parm_);\
- VOZ32(_op_, 11, ov, _nb_,_parm_);\
- VOZ32(_op_, 12, ov, _nb_,_parm_);\
- VOZ32(_op_, 13, ov, _nb_,_parm_);\
- VOZ32(_op_, 14, ov, _nb_,_parm_);\
- VOZ32(_op_, 15, ov, _nb_,_parm_);\
- VOZ32(_op_, 16, ov, _nb_,_parm_);\
- VOZ32(_op_, 17, ov, _nb_,_parm_);\
- VOZ32(_op_, 18, ov, _nb_,_parm_);\
- VOZ32(_op_, 19, ov, _nb_,_parm_);\
- VOZ32(_op_, 20, ov, _nb_,_parm_);\
- VOZ32(_op_, 21, ov, _nb_,_parm_);\
- VOZ32(_op_, 22, ov, _nb_,_parm_);\
- VOZ32(_op_, 23, ov, _nb_,_parm_);\
- VOZ32(_op_, 24, ov, _nb_,_parm_);\
- VOZ32(_op_, 25, ov, _nb_,_parm_);\
- VOZ32(_op_, 26, ov, _nb_,_parm_);\
- VOZ32(_op_, 27, ov, _nb_,_parm_);\
- VOZ32(_op_, 28, ov, _nb_,_parm_);\
- VOZ32(_op_, 29, ov, _nb_,_parm_);\
- VOZ32(_op_, 30, ov, _nb_,_parm_);\
- VOZ32(_op_, 31, ov, _nb_,_parm_);\
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm256_setzero_si256()
-
-unsigned char *bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv;
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-//--------------------------------------- zeromask unpack for TurboPFor vp4d.c --------------------------------------
-#define VO32(_op_, _i_, _ov_, _nb_,_parm_) xm = *bb++; _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) )); pex += popcnt32(xm)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_parm_) xm = *bb++; _mm256_storeu_si256(_op_++, mm256_maskz_loadu_epi32(xm,(__m256i*)pex) ); pex += popcnt32(xm)
-#define BITUNPACK0(_parm_)
-#include "bitunpack_.h"
-unsigned char *_bitunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm; __m256i sv, zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_)
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);//, zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);// zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_parm_) _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, sv))
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitfunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-//-----------------------------------------------------------------------------
-#define VX32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) ); pex += popcnt32(xm)
-#define VXZ32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = mm256_maskz_loadu_epi32(xm,(__m256i*)pex); pex += popcnt32(xm)
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitdunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm;
- __m256i sv = _mm256_set1_epi32(start), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7),zv = _mm256_setzero_si256();
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VX32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = _mm256_add_epi32(_ov_, _mm256_slli_epi32(mm256_maskz_loadu_epi32(xm,(__m256i*)pex), _nb_) ); pex += popcnt32(xm)
-#define VXZ32(_i_, _nb_,_ov_) xm = *bb++; _ov_ = mm256_maskz_loadu_epi32(xm,(__m256i*)pex); pex += popcnt32(xm)
-
- #ifdef _MSC_VER
-#define SCAN32x8( _v_, _sv_) {\
- _v_ = _mm256_add_epi32(_v_, _mm256_slli_si256(_v_, 4));\
- _v_ = _mm256_add_epi32(_v_, _mm256_slli_si256(_v_, 8));\
- _sv_ = _mm256_add_epi32( _mm256_permute2x128_si256( _mm256_shuffle_epi32(_sv_,_MM_SHUFFLE(3, 3, 3, 3)), _sv_, 0x11), \
- _mm256_add_epi32(_v_, _mm256_permute2x128_si256(zv,_mm256_shuffle_epi32(_v_, _MM_SHUFFLE(3, 3, 3, 3)), 0x20)));\
-}
-#define SCANI32x8(_v_, _sv_, _vi_) SCAN32x8(_v_, _sv_); _sv_ = _mm256_add_epi32(_sv_, _vi_)
-#define ZIGZAG32x8(_v_) _mm256_xor_si256(_mm256_slli_epi32(_v_,1), _mm256_srai_epi32(_v_,31))
-#define UNZIGZAG32x8(_v_) _mm256_xor_si256(_mm256_srli_epi32(_v_,1), _mm256_srai_epi32(_mm256_slli_epi32(_v_,31),31) )
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _ov_ = UNZIGZAG32x8(_ov_); SCAN32x8(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _ov_ = UNZIGZAG32x8(_ov_); SCAN32x8(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
- #else
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _ov_ = mm256_zzagd_epi32(_ov_); _sv_ = mm256_scan_epi32(_ov_,_sv_); _mm256_storeu_si256(_op_++, _sv_);
- #endif
-
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitzunpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b);
- unsigned xm;
- const __m256i zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm256_storeu_si256(_op_++, _parm_); _parm_ = _mm256_add_epi32(_parm_, cv)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) _parm_ = _mm256_add_epi32(_parm_, cv); cv = _mm256_set1_epi32(8)
-unsigned char *bitd1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- const __m256i zv = _mm256_setzero_si256();
- __m256i sv = _mm256_set1_epi32(start), cv = _mm256_set_epi32(8,7,6,5,4,3,2,1);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) _mm256_storeu_si256(_op_++, _mm256_add_epi32(_ov_, _sv_)); _sv_ = _mm256_add_epi32(_sv_, cv)
-#define VOZ32(_op_, _i_, ov, _nb_,_sv_) _mm256_storeu_si256(_op_++, _sv_); _sv_ = _mm256_add_epi32(_sv_, cv);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitf1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(256*b);
- const __m256i cv = _mm256_set1_epi32(8);
- __m256i sv = _mm256_set_epi32(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _sv_ = mm256_scani_epi32(_ov_,_sv_,cv); _mm256_storeu_si256(_op_++, _sv_);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) mv = _mm256_set1_epi32(0) //_parm_ = _mm_setzero_si128()
-unsigned char *_bitd1unpack256v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(256*b); unsigned xm;
- const __m256i cv = _mm256_set_epi32(8,7,6,5,4,3,2,1), zv = _mm256_setzero_si256(), tv = _mm256_set_epi32(0,1,2,3,4,5,6,7);
- __m256i sv = _mm256_set1_epi32(start);
- BITUNPACK256V32(in, b, out, sv);
- return (unsigned char *)ip;
-}
-
-size_t bitnunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; _BITNUNPACKV( in, n, out, 256, 32, bitunpack256v); }
-size_t bitndunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitdunpack256v, bitdunpack); }
-size_t bitnd1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitd1unpack256v, bitd1unpack); }
-//size_t bitns1unpack256v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bits1unpack256v, bitd1unpack); }
-size_t bitnzunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitzunpack256v, bitzunpack); }
-size_t bitnfunpack256v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 256, 32, bitfunpack256v, bitfunpack); }
- #elif defined(__SSE2__) || defined(__ARM_NEON) //------------------------------ SSE2/SSSE3 ---------------------------------------------------------
-#define BITMAX16 16
-#define BITMAX32 32
-
-#define VO16( _op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, ov)
-#define VO32( _op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, ov)
-#include "bitunpack_.h"
-
-#define VOZ16(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define BITUNBLK128V16_0(ip, _i_, _op_, _nb_,_parm_) {__m128i ov;\
- VOZ16(_op_, 0, ov, _nb_,_parm_);\
- VOZ16(_op_, 1, ov, _nb_,_parm_);\
- VOZ16(_op_, 2, ov, _nb_,_parm_);\
- VOZ16(_op_, 3, ov, _nb_,_parm_);\
- VOZ16(_op_, 4, ov, _nb_,_parm_);\
- VOZ16(_op_, 5, ov, _nb_,_parm_);\
- VOZ16(_op_, 6, ov, _nb_,_parm_);\
- VOZ16(_op_, 7, ov, _nb_,_parm_);\
- VOZ16(_op_, 8, ov, _nb_,_parm_);\
- VOZ16(_op_, 9, ov, _nb_,_parm_);\
- VOZ16(_op_, 10, ov, _nb_,_parm_);\
- VOZ16(_op_, 11, ov, _nb_,_parm_);\
- VOZ16(_op_, 12, ov, _nb_,_parm_);\
- VOZ16(_op_, 13, ov, _nb_,_parm_);\
- VOZ16(_op_, 14, ov, _nb_,_parm_);\
- VOZ16(_op_, 15, ov, _nb_,_parm_);\
- /*VOZ16(_op_, 16, ov, _nb_,_parm_);\
- VOZ16(_op_, 17, ov, _nb_,_parm_);\
- VOZ16(_op_, 18, ov, _nb_,_parm_);\
- VOZ16(_op_, 19, ov, _nb_,_parm_);\
- VOZ16(_op_, 20, ov, _nb_,_parm_);\
- VOZ16(_op_, 21, ov, _nb_,_parm_);\
- VOZ16(_op_, 22, ov, _nb_,_parm_);\
- VOZ16(_op_, 23, ov, _nb_,_parm_);\
- VOZ16(_op_, 24, ov, _nb_,_parm_);\
- VOZ16(_op_, 25, ov, _nb_,_parm_);\
- VOZ16(_op_, 26, ov, _nb_,_parm_);\
- VOZ16(_op_, 27, ov, _nb_,_parm_);\
- VOZ16(_op_, 28, ov, _nb_,_parm_);\
- VOZ16(_op_, 29, ov, _nb_,_parm_);\
- VOZ16(_op_, 30, ov, _nb_,_parm_);\
- VOZ16(_op_, 31, ov, _nb_,_parm_);*/\
-}
-
-#define BITUNBLK128V32_0(ip, _i_, _op_, _nb_,_parm_) {__m128i ov;\
- VOZ32(_op_, 0, ov, _nb_,_parm_);\
- VOZ32(_op_, 1, ov, _nb_,_parm_);\
- VOZ32(_op_, 2, ov, _nb_,_parm_);\
- VOZ32(_op_, 3, ov, _nb_,_parm_);\
- VOZ32(_op_, 4, ov, _nb_,_parm_);\
- VOZ32(_op_, 5, ov, _nb_,_parm_);\
- VOZ32(_op_, 6, ov, _nb_,_parm_);\
- VOZ32(_op_, 7, ov, _nb_,_parm_);\
- VOZ32(_op_, 8, ov, _nb_,_parm_);\
- VOZ32(_op_, 9, ov, _nb_,_parm_);\
- VOZ32(_op_, 10, ov, _nb_,_parm_);\
- VOZ32(_op_, 11, ov, _nb_,_parm_);\
- VOZ32(_op_, 12, ov, _nb_,_parm_);\
- VOZ32(_op_, 13, ov, _nb_,_parm_);\
- VOZ32(_op_, 14, ov, _nb_,_parm_);\
- VOZ32(_op_, 15, ov, _nb_,_parm_);\
- VOZ32(_op_, 16, ov, _nb_,_parm_);\
- VOZ32(_op_, 17, ov, _nb_,_parm_);\
- VOZ32(_op_, 18, ov, _nb_,_parm_);\
- VOZ32(_op_, 19, ov, _nb_,_parm_);\
- VOZ32(_op_, 20, ov, _nb_,_parm_);\
- VOZ32(_op_, 21, ov, _nb_,_parm_);\
- VOZ32(_op_, 22, ov, _nb_,_parm_);\
- VOZ32(_op_, 23, ov, _nb_,_parm_);\
- VOZ32(_op_, 24, ov, _nb_,_parm_);\
- VOZ32(_op_, 25, ov, _nb_,_parm_);\
- VOZ32(_op_, 26, ov, _nb_,_parm_);\
- VOZ32(_op_, 27, ov, _nb_,_parm_);\
- VOZ32(_op_, 28, ov, _nb_,_parm_);\
- VOZ32(_op_, 29, ov, _nb_,_parm_);\
- VOZ32(_op_, 30, ov, _nb_,_parm_);\
- VOZ32(_op_, 31, ov, _nb_,_parm_);\
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm_setzero_si128()
-
-unsigned char *bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b) { const unsigned char *ip = in+PAD8(128*b); __m128i sv; BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip; }
-unsigned char *bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) { const unsigned char *ip = in+PAD8(128*b); __m128i sv; BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip; }
-unsigned char *bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b) {
- const unsigned char *_in=in; unsigned *_out=out; __m128i sv;
- BITUNPACK128V32(in, b, out, sv); out = _out+128; in=_in+PAD8(128*b);
- BITUNPACK128V32(in, b, out, sv);
- return (unsigned char *)_in+PAD8(256*b);
-}
-
-#define STOZ64(_op_, _ov_) _mm_storeu_si128(_op_++, _ov_); _mm_storeu_si128(_op_++, _ov_)
-#define STO64( _op_, _ov_, _zv_) _mm_storeu_si128(_op_++, _mm_unpacklo_epi32(_ov_,_zv_));_mm_storeu_si128(_op_++, _mm_unpacklo_epi32(_mm_srli_si128(_ov_,8),_zv_))
-
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) STOZ64(_op_, _parm_)
-#define VO32( _op_, _i_, ov, _nb_,_parm_) STO64(_op_, ov, zv)
-#include "bitunpack_.h"
-unsigned char *bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b) {
- if(b <= 32) { const unsigned char *ip = in+PAD8(128*b);
- __m128i sv,zv = _mm_setzero_si128();
- BITUNPACK128V32(in, b, out, sv);
- return (unsigned char *)ip;
- } else return bitunpack64(in,n,out,b);
-}
-#undef VO32
-#undef VOZ32
-#undef VO16
-#undef VOZ16
-#undef BITUNPACK0
-
- #if defined(__SSSE3__) || defined(__ARM_NEON)
- #define _ 0x80
-ALIGNED(char, _shuffle_32[16][16],16) = {
- { _,_,_,_, _,_,_,_, _,_, _, _, _, _, _,_ },
- { 0,1,2,3, _,_,_,_, _,_, _, _, _, _, _,_ },
- { _,_,_,_, 0,1,2,3, _,_, _, _, _, _, _,_ },
- { 0,1,2,3, 4,5,6,7, _,_, _, _, _, _, _,_ },
- { _,_,_,_, _,_,_,_, 0,1, 2, 3, _, _, _,_ },
- { 0,1,2,3, _,_,_,_, 4,5, 6, 7, _, _, _,_ },
- { _,_,_,_, 0,1,2,3, 4,5, 6, 7, _, _, _,_ },
- { 0,1,2,3, 4,5,6,7, 8,9,10,11, _, _, _,_ },
- { _,_,_,_, _,_,_,_, _,_,_,_, 0, 1, 2, 3 },
- { 0,1,2,3, _,_,_,_, _,_,_, _, 4, 5, 6, 7 },
- { _,_,_,_, 0,1,2,3, _,_,_, _, 4, 5, 6, 7 },
- { 0,1,2,3, 4,5,6,7, _,_, _, _, 8, 9,10,11 },
- { _,_,_,_, _,_,_,_, 0,1, 2, 3, 4, 5, 6, 7 },
- { 0,1,2,3, _,_,_,_, 4,5, 6, 7, 8, 9,10,11 },
- { _,_,_,_, 0,1,2,3, 4,5, 6, 7, 8, 9,10,11 },
- { 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15 },
-};
-ALIGNED(char, _shuffle_16[256][16],16) = {
- { _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ },
- { 0, 1, _, _, _, _, _, _, _, _, _, _, _, _, _, _ },
- { _, _, 0, 1, _, _, _, _, _, _, _, _, _, _, _, _ },
- { 0, 1, 2, 3, _, _, _, _, _, _, _, _, _, _, _, _ },
- { _, _, _, _, 0, 1, _, _, _, _, _, _, _, _, _, _ },
- { 0, 1, _, _, 2, 3, _, _, _, _, _, _, _, _, _, _ },
- { _, _, 0, 1, 2, 3, _, _, _, _, _, _, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, _, _, _, _ },
- { _, _, _, _, _, _, 0, 1, _, _, _, _, _, _, _, _ },
- { 0, 1, _, _, _, _, 2, 3, _, _, _, _, _, _, _, _ },
- { _, _, 0, 1, _, _, 2, 3, _, _, _, _, _, _, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, _, _, _, _ },
- { _, _, _, _, 0, 1, 2, 3, _, _, _, _, _, _, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, _, _, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, _, _, _, _ },
- { _, _, _, _, _, _, _, _, 0, 1, _, _, _, _, _, _ },
- { 0, 1, _, _, _, _, _, _, 2, 3, _, _, _, _, _, _ },
- { _, _, 0, 1, _, _, _, _, 2, 3, _, _, _, _, _, _ },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, _, _, _, _ },
- { _, _, _, _, 0, 1, _, _, 2, 3, _, _, _, _, _, _ },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, _, _, _, _ },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, _, _, _, _ },
- { _, _, _, _, _, _, 0, 1, 2, 3, _, _, _, _, _, _ },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, _, _, _, _ },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, _, _, _, _ },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, _, _, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _, _, _, _, _ },
- { _, _, _, _, _, _, _, _, _, _, 0, 1, _, _, _, _ },
- { 0, 1, _, _, _, _, _, _, _, _, 2, 3, _, _, _, _ },
- { _, _, 0, 1, _, _, _, _, _, _, 2, 3, _, _, _, _ },
- { 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, _, _, _, _ },
- { _, _, _, _, 0, 1, _, _, _, _, 2, 3, _, _, _, _ },
- { 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, _, _, _, _ },
- { _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, _, _, _, _ },
- { _, _, _, _, _, _, 0, 1, _, _, 2, 3, _, _, _, _ },
- { 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, _, _, _, _ },
- { _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, _, _, _, _ },
- { _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, _, _, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9, _, _, _, _ },
- { _, _, _, _, _, _, _, _, 0, 1, 2, 3, _, _, _, _ },
- { 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, _, _, _, _ },
- { _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, _, _ },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, _, _, _, _ },
- { _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, _, _ },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, _, _, _, _ },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9, _, _, _, _ },
- { _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, _, _ },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, _, _, _, _ },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9, _, _, _, _ },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9, _, _, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, _, _, _, _ },
- { _, _, _, _, _, _, _, _, _, _, _, _, 0, 1, _, _ },
- { 0, 1, _, _, _, _, _, _, _, _, _, _, 2, 3, _, _ },
- { _, _, 0, 1, _, _, _, _, _, _, _, _, 2, 3, _, _ },
- { 0, 1, 2, 3, _, _, _, _, _, _, _, _, 4, 5, _, _ },
- { _, _, _, _, 0, 1, _, _, _, _, _, _, 2, 3, _, _ },
- { 0, 1, _, _, 2, 3, _, _, _, _, _, _, 4, 5, _, _ },
- { _, _, 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, 6, 7, _, _ },
- { _, _, _, _, _, _, 0, 1, _, _, _, _, 2, 3, _, _ },
- { 0, 1, _, _, _, _, 2, 3, _, _, _, _, 4, 5, _, _ },
- { _, _, 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, 6, 7, _, _ },
- { _, _, _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, 6, 7, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, 8, 9, _, _ },
- { _, _, _, _, _, _, _, _, 0, 1, _, _, 2, 3, _, _ },
- { 0, 1, _, _, _, _, _, _, 2, 3, _, _, 4, 5, _, _ },
- { _, _, 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, _, _ },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, 6, 7, _, _ },
- { _, _, _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, _, _ },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, 6, 7, _, _ },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, 8, 9, _, _ },
- { _, _, _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, _, _ },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, 6, 7, _, _ },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, 8, 9, _, _ },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, 8, 9, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _,10,11, _, _ },
- { _, _, _, _, _, _, _, _, _, _, 0, 1, 2, 3, _, _ },
- { 0, 1, _, _, _, _, _, _, _, _, 2, 3, 4, 5, _, _ },
- { _, _, 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, _, _ },
- { 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, 6, 7, _, _ },
- { _, _, _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, _, _ },
- { 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, 6, 7, _, _ },
- { _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, 8, 9, _, _ },
- { _, _, _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, _, _ },
- { 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, 6, 7, _, _ },
- { _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, 8, 9, _, _ },
- { _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, 8, 9, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9,10,11, _, _ },
- { _, _, _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, _, _ },
- { 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, 6, 7, _, _ },
- { _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, _, _ },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, 8, 9, _, _ },
- { _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _ },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, 8, 9, _, _ },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9, _, _ },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9,10,11, _, _ },
- { _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _ },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, 8, 9, _, _ },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9, _, _ },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9,10,11, _, _ },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _ },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9,10,11, _, _ },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, _, _ },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, _, _ },
- { _, _, _, _, _, _, _, _, _, _, _, _, _, _, 0, 1 },
- { 0, 1, _, _, _, _, _, _, _, _, _, _, _, _, 2, 3 },
- { _, _, 0, 1, _, _, _, _, _, _, _, _, _, _, 2, 3 },
- { 0, 1, 2, 3, _, _, _, _, _, _, _, _, _, _, 4, 5 },
- { _, _, _, _, 0, 1, _, _, _, _, _, _, _, _, 2, 3 },
- { 0, 1, _, _, 2, 3, _, _, _, _, _, _, _, _, 4, 5 },
- { _, _, 0, 1, 2, 3, _, _, _, _, _, _, _, _, 4, 5 },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, _, _, 6, 7 },
- { _, _, _, _, _, _, 0, 1, _, _, _, _, _, _, 2, 3 },
- { 0, 1, _, _, _, _, 2, 3, _, _, _, _, _, _, 4, 5 },
- { _, _, 0, 1, _, _, 2, 3, _, _, _, _, _, _, 4, 5 },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, _, _, 6, 7 },
- { _, _, _, _, 0, 1, 2, 3, _, _, _, _, _, _, 4, 5 },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, _, _, 6, 7 },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, 6, 7 },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, _, _, 8, 9 },
- { _, _, _, _, _, _, _, _, 0, 1, _, _, _, _, 2, 3 },
- { 0, 1, _, _, _, _, _, _, 2, 3, _, _, _, _, 4, 5 },
- { _, _, 0, 1, _, _, _, _, 2, 3, _, _, _, _, 4, 5 },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, _, _, 6, 7 },
- { _, _, _, _, 0, 1, _, _, 2, 3, _, _, _, _, 4, 5 },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, _, _, 6, 7 },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, 6, 7 },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, _, _, 8, 9 },
- { _, _, _, _, _, _, 0, 1, 2, 3, _, _, _, _, 4, 5 },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, _, _, 6, 7 },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, 6, 7 },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, _, _, 8, 9 },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7 },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, _, _, 8, 9 },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, 8, 9 },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _, _, _,10,11 },
- { _, _, _, _, _, _, _, _, _, _, 0, 1, _, _, 2, 3 },
- { 0, 1, _, _, _, _, _, _, _, _, 2, 3, _, _, 4, 5 },
- { _, _, 0, 1, _, _, _, _, _, _, 2, 3, _, _, 4, 5 },
- { 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, _, _, 6, 7 },
- { _, _, _, _, 0, 1, _, _, _, _, 2, 3, _, _, 4, 5 },
- { 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, _, _, 6, 7 },
- { _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, 6, 7 },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, _, _, 8, 9 },
- { _, _, _, _, _, _, 0, 1, _, _, 2, 3, _, _, 4, 5 },
- { 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, _, _, 6, 7 },
- { _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, 6, 7 },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, _, _, 8, 9 },
- { _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7 },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, _, _, 8, 9 },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, 8, 9 },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9, _, _,10,11 },
- { _, _, _, _, _, _, _, _, 0, 1, 2, 3, _, _, 4, 5 },
- { 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, _, _, 6, 7 },
- { _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, 6, 7 },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, _, _, 8, 9 },
- { _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7 },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, _, _, 8, 9 },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, 8, 9 },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9, _, _,10,11 },
- { _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7 },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, _, _, 8, 9 },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, 8, 9 },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9, _, _,10,11 },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9 },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9, _, _,10,11 },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _,10,11 },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, _, _,12,13 },
- { _, _, _, _, _, _, _, _, _, _, _, _, 0, 1, 2, 3 },
- { 0, 1, _, _, _, _, _, _, _, _, _, _, 2, 3, 4, 5 },
- { _, _, 0, 1, _, _, _, _, _, _, _, _, 2, 3, 4, 5 },
- { 0, 1, 2, 3, _, _, _, _, _, _, _, _, 4, 5, 6, 7 },
- { _, _, _, _, 0, 1, _, _, _, _, _, _, 2, 3, 4, 5 },
- { 0, 1, _, _, 2, 3, _, _, _, _, _, _, 4, 5, 6, 7 },
- { _, _, 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, 6, 7 },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, _, _, 6, 7, 8, 9 },
- { _, _, _, _, _, _, 0, 1, _, _, _, _, 2, 3, 4, 5 },
- { 0, 1, _, _, _, _, 2, 3, _, _, _, _, 4, 5, 6, 7 },
- { _, _, 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, 6, 7 },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, _, _, 6, 7, 8, 9 },
- { _, _, _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7 },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, _, _, 6, 7, 8, 9 },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, 8, 9 },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, _, _, 8, 9,10,11 },
- { _, _, _, _, _, _, _, _, 0, 1, _, _, 2, 3, 4, 5 },
- { 0, 1, _, _, _, _, _, _, 2, 3, _, _, 4, 5, 6, 7 },
- { _, _, 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, 6, 7 },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, _, _, 6, 7, 8, 9 },
- { _, _, _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7 },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, _, _, 6, 7, 8, 9 },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, 8, 9 },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, _, _, 8, 9,10,11 },
- { _, _, _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7 },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, _, _, 6, 7, 8, 9 },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, 8, 9 },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, _, _, 8, 9,10,11 },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9 },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, _, _, 8, 9,10,11 },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9,10,11 },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, _, _,10,11,12,13 },
- { _, _, _, _, _, _, _, _, _, _, 0, 1, 2, 3, 4, 5 },
- { 0, 1, _, _, _, _, _, _, _, _, 2, 3, 4, 5, 6, 7 },
- { _, _, 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, 6, 7 },
- { 0, 1, 2, 3, _, _, _, _, _, _, 4, 5, 6, 7, 8, 9 },
- { _, _, _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7 },
- { 0, 1, _, _, 2, 3, _, _, _, _, 4, 5, 6, 7, 8, 9 },
- { _, _, 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, 8, 9 },
- { 0, 1, 2, 3, 4, 5, _, _, _, _, 6, 7, 8, 9,10,11 },
- { _, _, _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7 },
- { 0, 1, _, _, _, _, 2, 3, _, _, 4, 5, 6, 7, 8, 9 },
- { _, _, 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, 8, 9 },
- { 0, 1, 2, 3, _, _, 4, 5, _, _, 6, 7, 8, 9,10,11 },
- { _, _, _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9 },
- { 0, 1, _, _, 2, 3, 4, 5, _, _, 6, 7, 8, 9,10,11 },
- { _, _, 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9,10,11 },
- { 0, 1, 2, 3, 4, 5, 6, 7, _, _, 8, 9,10,11,12,13 },
- { _, _, _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7 },
- { 0, 1, _, _, _, _, _, _, 2, 3, 4, 5, 6, 7, 8, 9 },
- { _, _, 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, 8, 9 },
- { 0, 1, 2, 3, _, _, _, _, 4, 5, 6, 7, 8, 9,10,11 },
- { _, _, _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9 },
- { 0, 1, _, _, 2, 3, _, _, 4, 5, 6, 7, 8, 9,10,11 },
- { _, _, 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9,10,11 },
- { 0, 1, 2, 3, 4, 5, _, _, 6, 7, 8, 9,10,11,12,13 },
- { _, _, _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 },
- { 0, 1, _, _, _, _, 2, 3, 4, 5, 6, 7, 8, 9,10,11 },
- { _, _, 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9,10,11 },
- { 0, 1, 2, 3, _, _, 4, 5, 6, 7, 8, 9,10,11,12,13 },
- { _, _, _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11 },
- { 0, 1, _, _, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13 },
- { _, _, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13 },
- { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15 },
-};
- #undef _
- #endif // SSSE3
-
-#define BITMAX16 15
-#define BITMAX32 31
-
-#define VO16( _op_, _i_, _ov_, _nb_,_parm_) m = *bb++; _mm_storeu_si128(_op_++, _mm_add_epi16(_ov_, _mm_shuffle_epi8( mm_slli_epi16(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ) )); pex += popcnt32(m)
-#define VO32( _op_, _i_, _ov_, _nb_,_parm_) if((_i_) & 1) m = (*bb++) >> 4; else m = (*bb) & 0xf; _mm_storeu_si128(_op_++, _mm_add_epi32(_ov_, _mm_shuffle_epi8( mm_slli_epi32(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ) )); pex += popcnt32(m)
-#define VOZ16(_op_, _i_, _ov_, _nb_,_parm_) m = *bb++; _mm_storeu_si128(_op_++, _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ) ); pex += popcnt32(m)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_parm_) if((_i_) & 1) m = (*bb++) >> 4; else m = (*bb) & 0xf; _mm_storeu_si128(_op_++, _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ) ); pex += popcnt32(m)
-#define BITUNPACK0(_parm_) //_parm_ = _mm_setzero_si128()
-#include "bitunpack_.h"
-
-unsigned char *_bitunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned b, unsigned short *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv; BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *_bitunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv; BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *_bitunpack256w32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *_in=in; unsigned *_out=out, m; __m128i sv;
- BITUNPACK128V32(in, b, out, sv); out = _out+128; in=_in+PAD8(128*b);
- BITUNPACK128V32(in, b, out, sv);
- return (unsigned char *)_in+PAD8(256*b);
-}
-
-//#define STOZ64(_op_, _ov_) _mm_storeu_si128(_op_++, _ov_); _mm_storeu_si128(_op_++, _ov_)
-#define STO64( _op_, _ov_, _zv_) _mm_storeu_si128(_op_++, _mm_unpacklo_epi32(_ov_,_zv_));_mm_storeu_si128(_op_++, _mm_unpacklo_epi32(_mm_srli_si128(_ov_,8),_zv_))
-
-#define VO32( _op_, _i_, _ov_, _nb_,_parm_) if((_i_) & 1) m = (*bb++) >> 4; else m = (*bb) & 0xf; { __m128i _wv = _mm_add_epi32(_ov_, _mm_shuffle_epi8( mm_slli_epi32(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ) ); STO64(_op_, _wv, zv);} pex += popcnt32(m)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_parm_) if((_i_) & 1) m = (*bb++) >> 4; else m = (*bb) & 0xf; { __m128i _wv = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ) ; STO64(_op_, _wv, zv);} pex += popcnt32(m)
-#define BITUNPACK0(_parm_)
-
-#include "bitunpack_.h"
-unsigned char *_bitunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, unsigned b, uint32_t *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i zv = _mm_setzero_si128(); BITUNPACK128V32(in, b, out, 0); return (unsigned char *)ip;
-}
-
-#define BITMAX16 16
-#define BITMAX32 32
-
-#undef VO32
-#undef VOZ32
-#undef VO16
-#undef VOZ16
-#undef BITUNPACK0
-
-//-------------------------------------------------------------------
-#define VOZ16(_op_, _i_, _ov_, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_)
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) _ov_ = mm_zzagd_epi16(_ov_); _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_)
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) _ov_ = mm_zzagd_epi32(_ov_); _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi16(start); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi32(start); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_)
-#define VO16(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitdunpack128v16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi16(start); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi32(start); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-#define VO32( _op_, _i_, _ov_, _nb_,_parm_) _mm_storeu_si128(_op_++, _mm_add_epi32(_ov_, sv))
-#define VO16( _op_, _i_, _ov_, _nb_,_parm_) _mm_storeu_si128(_op_++, _mm_add_epi16(_ov_, sv))
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitfunpack128v16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi16(start); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *bitfunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi32(start); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
- #if defined(__SSSE3__) || defined(__ARM_NEON)
-#define BITMAX16 15
-#define BITMAX32 31
-
-#define VX32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_add_epi32(_ov_, _mm_shuffle_epi8( mm_slli_epi32(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_32[m]))); pex += popcnt32(m)
-#define VXZ32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_32[m])); pex += popcnt32(m)
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-
-#define VX16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_add_epi16(_ov_, _mm_shuffle_epi8( mm_slli_epi16(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ) ); pex += popcnt32(m)
-#define VXZ16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ); pex += popcnt32(m)
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) VX16( _i_, _nb_,_ov_); _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, _ov_, _nb_,_sv_) VXZ16( _i_, _nb_,_ov_); _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitdunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi16(start); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *_bitdunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi32(start); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-/*
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _ov_); mm_scan_epi32(_ov_,_sv_); STO64( _op_, _sv_) //_mm_storeu_si128(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32( _i_, _ov_); mm_scan_epi32(_ov_,_sv_); STOZ64( _op_, _sv_, zv) //_mm_storeu_si128(_op_++, _sv_);
-unsigned char *_bitdunpack128v64( const unsigned char *__restrict in, unsigned n, uint64_t *__restrict out, uint64_t start, unsigned b, uint64_t *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi32(start),zv = _mm_setzero_si128(); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}*/
-
-#define VX16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_add_epi16(_ov_, _mm_shuffle_epi8( mm_slli_epi16(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ) ); pex += popcnt32(m)
-#define VXZ16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_16[m]) ); pex += popcnt32(m)
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) VX16( _i_, _nb_,_ov_); _ov_ = mm_zzagd_epi16(_ov_); _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, _ov_, _nb_,_sv_) VXZ16( _i_, _nb_,_ov_); _ov_ = mm_zzagd_epi16(_ov_); _sv_ = mm_scan_epi16(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-
-#define VX32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_add_epi32(_ov_, _mm_shuffle_epi8( mm_slli_epi32(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ) ); pex += popcnt32(m)
-#define VXZ32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_32[m]) ); pex += popcnt32(m)
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _ov_ = mm_zzagd_epi32(_ov_); _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32(_i_, _nb_,_ov_); _ov_ = mm_zzagd_epi32(_ov_); _sv_ = mm_scan_epi32(_ov_,_sv_); _mm_storeu_si128(_op_++, _sv_);
-
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *_bitzunpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi16(start); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *_bitzunpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi32(start); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITMAX16 16
-#define BITMAX32 32
- #endif
-
-#define VO16(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm_scani_epi16(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VO32(_op_, i, _ov_, _nb_,_sv_) _sv_ = mm_scani_epi32(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_); _parm_ = _mm_add_epi16(_parm_, cv)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_); _parm_ = _mm_add_epi32(_parm_, cv)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) _parm_ = _mm_add_epi16(_parm_, cv); cv = _mm_set1_epi16(8)
-unsigned char *bitd1unpack128v16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi16(start), cv = _mm_set_epi16(8,7,6,5,4,3,2,1); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm_add_epi32(_parm_, cv); cv = _mm_set1_epi32(4)
-unsigned char *bitd1unpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi32(start), cv = _mm_set_epi32(4,3,2,1); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-#define VO16(_op_, i, _ov_, _nb_,_sv_) ADDI16x8(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VO32(_op_, i, _ov_, _nb_,_sv_) ADDI32x4(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_); _parm_ = _mm_add_epi16(_parm_, cv)
-#define VOZ32(_op_, _i_, ov, _nb_,_parm_) _mm_storeu_si128(_op_++, _parm_); _parm_ = _mm_add_epi32(_parm_, cv)
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) _parm_ = _mm_add_epi16(_parm_, cv); cv = _mm_set1_epi16(8)
-unsigned char *bits1unpack128v16( const unsigned char *__restrict in, unsigned n, uint16_t *__restrict out, uint16_t start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(8); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITUNPACK0(_parm_) _parm_ = _mm_add_epi32(_parm_, cv); cv = _mm_set1_epi32(4)
-unsigned char *bits1unpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(4); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) _mm_storeu_si128(_op_++, _mm_add_epi16(_ov_, _sv_)); _sv_ = _mm_add_epi16(_sv_, cv)
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) _mm_storeu_si128(_op_++, _mm_add_epi32(_ov_, _sv_)); _sv_ = _mm_add_epi32(_sv_, cv)
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) _mm_storeu_si128(_op_++, _sv_); _sv_ = _mm_add_epi32(_sv_, cv);
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_)
-unsigned char *bitf1unpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set_epi16(start+8,start+7,start+6,start+5,start+4,start+3,start+2,start+1), cv = _mm_set1_epi16(8); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-unsigned char *bitf1unpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
- const unsigned char *ip = in+PAD8(128*b); __m128i sv = _mm_set_epi32(start+4,start+3,start+2,start+1), cv = _mm_set1_epi32(4); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
- #if defined(__SSSE3__) || defined(__ARM_NEON)
-#define BITMAX16 15
-#define BITMAX32 31
-
-#define VX16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_add_epi16(_ov_, _mm_shuffle_epi8( mm_slli_epi16(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_16[m]))); pex += popcnt32(m)
-#define VX32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_add_epi32(_ov_, _mm_shuffle_epi8( mm_slli_epi32(_mm_loadu_si128((__m128i*)pex), _nb_), _mm_loadu_si128((__m128i*)_shuffle_32[m]))); pex += popcnt32(m)
-#define VXZ16(_i_, _nb_,_ov_) m = *bb++; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_16[m])); pex += popcnt32(m)
-#define VXZ32(_i_, _nb_,_ov_) if(!((_i_) & 1)) m = (*bb) & 0xf;else m = (*bb++) >> 4; _ov_ = _mm_shuffle_epi8( _mm_loadu_si128((__m128i*)pex), _mm_loadu_si128((__m128i*)_shuffle_32[m])); pex += popcnt32(m)
-
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) VX16( _i_, _nb_,_ov_); _sv_ = mm_scani_epi16(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, _ov_, _nb_,_sv_) VXZ16( _i_, _nb_,_ov_); _sv_ = mm_scani_epi16(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); _sv_ = mm_scani_epi32(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32( _i_, _nb_,_ov_); _sv_ = mm_scani_epi32(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) mv = _mm_setzero_si128() //_parm_ = _mm_setzero_si128()
-unsigned char *_bitd1unpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi16(start), cv = _mm_set_epi16(8,7,6,5,4,3,2,1); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITUNPACK0(_parm_) mv = _mm_setzero_si128()
-unsigned char *_bitd1unpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi32(start), cv = _mm_set_epi32( 4,3,2,1); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-
-#define VO16( _op_, _i_, _ov_, _nb_,_sv_) VX16( _i_, _nb_,_ov_); ADDI16x8(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ16(_op_, _i_, _ov_, _nb_,_sv_) VXZ16( _i_, _nb_,_ov_); ADDI16x8(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VO32( _op_, _i_, _ov_, _nb_,_sv_) VX32( _i_, _nb_,_ov_); ADDI32x4(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-#define VOZ32(_op_, _i_, _ov_, _nb_,_sv_) VXZ32( _i_, _nb_,_ov_); ADDI32x4(_ov_,_sv_,cv); _mm_storeu_si128(_op_++, _sv_);
-
-#include "bitunpack_.h"
-#define BITUNPACK0(_parm_) mv = _mm_setzero_si128() //_parm_ = _mm_setzero_si128()
-unsigned char *_bits1unpack128v16( const unsigned char *__restrict in, unsigned n, unsigned short *__restrict out, unsigned short start, unsigned b, unsigned short *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi16(start), cv = _mm_set1_epi16(8); BITUNPACK128V16(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITUNPACK0(_parm_) mv = _mm_setzero_si128()
-unsigned char *_bits1unpack128v32( const unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b, unsigned *__restrict pex, unsigned char *bb) {
- const unsigned char *ip = in+PAD8(128*b); unsigned m; __m128i sv = _mm_set1_epi32(start), cv = _mm_set1_epi32(4); BITUNPACK128V32(in, b, out, sv); return (unsigned char *)ip;
-}
-#define BITMAX16 16
-#define BITMAX32 32
- #endif
-
-size_t bitnunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op; _BITNUNPACKV( in, n, out, 128, 16, bitunpack128v); }
-size_t bitnunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; _BITNUNPACKV( in, n, out, 128, 32, bitunpack128v); }
-size_t bitnunpack128v64( unsigned char *__restrict in, size_t n, uint64_t *__restrict out) { uint64_t *op; _BITNUNPACKV( in, n, out, 128, 64, bitunpack128v); }
-size_t bitnunpack256w32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op; _BITNUNPACKV( in, n, out, 256, 32, bitunpack256w); }
-
-size_t bitndunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; _BITNDUNPACKV(in, n, out, 128, 16, bitdunpack128v, bitdunpack); }
-size_t bitndunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 128, 32, bitdunpack128v, bitdunpack); }
-
-size_t bitnd1unpack128v16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; _BITNDUNPACKV(in, n, out, 128, 16, bitd1unpack128v, bitd1unpack); }
-size_t bitnd1unpack128v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 128, 32, bitd1unpack128v, bitd1unpack); }
-
-size_t bitns1unpack128v16(unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; _BITNDUNPACKV(in, n, out, 128, 16, bits1unpack128v, bitd1unpack); }
-size_t bitns1unpack128v32(unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 128, 32, bits1unpack128v, bitd1unpack); }
-
-size_t bitnzunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; _BITNDUNPACKV(in, n, out, 128, 16, bitzunpack128v, bitzunpack); }
-size_t bitnzunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 128, 32, bitzunpack128v, bitzunpack); }
-
-size_t bitnfunpack128v16( unsigned char *__restrict in, size_t n, uint16_t *__restrict out) { uint16_t *op,start; _BITNDUNPACKV(in, n, out, 128, 16, bitfunpack128v, bitfunpack); }
-size_t bitnfunpack128v32( unsigned char *__restrict in, size_t n, uint32_t *__restrict out) { uint32_t *op,start; _BITNDUNPACKV(in, n, out, 128, 32, bitfunpack128v, bitfunpack); }
-
-#endif
-#endif
-
-#pragma clang diagnostic pop
-#pragma GCC pop_options
diff --git a/src/ext/for/bitunpack_.h b/src/ext/for/bitunpack_.h
deleted file mode 100644
index cebbbe9f6fc..00000000000
--- a/src/ext/for/bitunpack_.h
+++ /dev/null
@@ -1,6049 +0,0 @@
-/**
- Copyright (C) powturbo 2013-2017
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- - homepage : https://sites.google.com/site/powturbo/
- - github : https://github.com/powturbo
- - twitter : https://twitter.com/powturbo
- - email : powturbo [_AT_] gmail [_DOT_] com
-**/
-// TurboPFor Integer Compression: Bit Packing
-#ifdef OPI
-#define BITUNBLK64_0(ip, i, op, nb,parm) { \
- OUT(op,i*0+ 0, 0, nb,parm);\
- OUT(op,i*0+ 1, 0, nb,parm);\
- OUT(op,i*0+ 2, 0, nb,parm);\
- OUT(op,i*0+ 3, 0, nb,parm);\
- OUT(op,i*0+ 4, 0, nb,parm);\
- OUT(op,i*0+ 5, 0, nb,parm);\
- OUT(op,i*0+ 6, 0, nb,parm);\
- OUT(op,i*0+ 7, 0, nb,parm);\
- OUT(op,i*0+ 8, 0, nb,parm);\
- OUT(op,i*0+ 9, 0, nb,parm);\
- OUT(op,i*0+10, 0, nb,parm);\
- OUT(op,i*0+11, 0, nb,parm);\
- OUT(op,i*0+12, 0, nb,parm);\
- OUT(op,i*0+13, 0, nb,parm);\
- OUT(op,i*0+14, 0, nb,parm);\
- OUT(op,i*0+15, 0, nb,parm);\
- OUT(op,i*0+16, 0, nb,parm);\
- OUT(op,i*0+17, 0, nb,parm);\
- OUT(op,i*0+18, 0, nb,parm);\
- OUT(op,i*0+19, 0, nb,parm);\
- OUT(op,i*0+20, 0, nb,parm);\
- OUT(op,i*0+21, 0, nb,parm);\
- OUT(op,i*0+22, 0, nb,parm);\
- OUT(op,i*0+23, 0, nb,parm);\
- OUT(op,i*0+24, 0, nb,parm);\
- OUT(op,i*0+25, 0, nb,parm);\
- OUT(op,i*0+26, 0, nb,parm);\
- OUT(op,i*0+27, 0, nb,parm);\
- OUT(op,i*0+28, 0, nb,parm);\
- OUT(op,i*0+29, 0, nb,parm);\
- OUT(op,i*0+30, 0, nb,parm);\
- OUT(op,i*0+31, 0, nb,parm);;\
-}
-
-#define BITUNPACK64_0(ip, op, nb,parm) { \
- BITUNBLK64_0(ip, 0, op, nb,parm); OPI(op, nb,parm);\
-}
-
-#define BITUNBLK64_1(ip, i, op, nb,parm) { uint32_t w0; w0 = *(uint32_t *)(ip+(i*1+0)*4/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x1, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 1) & 0x1, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 2) & 0x1, nb,parm);\
- OUT(op,i*32+ 3, (w0 >> 3) & 0x1, nb,parm);\
- OUT(op,i*32+ 4, (w0 >> 4) & 0x1, nb,parm);\
- OUT(op,i*32+ 5, (w0 >> 5) & 0x1, nb,parm);\
- OUT(op,i*32+ 6, (w0 >> 6) & 0x1, nb,parm);\
- OUT(op,i*32+ 7, (w0 >> 7) & 0x1, nb,parm);\
- OUT(op,i*32+ 8, (w0 >> 8) & 0x1, nb,parm);\
- OUT(op,i*32+ 9, (w0 >> 9) & 0x1, nb,parm);\
- OUT(op,i*32+10, (w0 >> 10) & 0x1, nb,parm);\
- OUT(op,i*32+11, (w0 >> 11) & 0x1, nb,parm);\
- OUT(op,i*32+12, (w0 >> 12) & 0x1, nb,parm);\
- OUT(op,i*32+13, (w0 >> 13) & 0x1, nb,parm);\
- OUT(op,i*32+14, (w0 >> 14) & 0x1, nb,parm);\
- OUT(op,i*32+15, (w0 >> 15) & 0x1, nb,parm);\
- OUT(op,i*32+16, (w0 >> 16) & 0x1, nb,parm);\
- OUT(op,i*32+17, (w0 >> 17) & 0x1, nb,parm);\
- OUT(op,i*32+18, (w0 >> 18) & 0x1, nb,parm);\
- OUT(op,i*32+19, (w0 >> 19) & 0x1, nb,parm);\
- OUT(op,i*32+20, (w0 >> 20) & 0x1, nb,parm);\
- OUT(op,i*32+21, (w0 >> 21) & 0x1, nb,parm);\
- OUT(op,i*32+22, (w0 >> 22) & 0x1, nb,parm);\
- OUT(op,i*32+23, (w0 >> 23) & 0x1, nb,parm);\
- OUT(op,i*32+24, (w0 >> 24) & 0x1, nb,parm);\
- OUT(op,i*32+25, (w0 >> 25) & 0x1, nb,parm);\
- OUT(op,i*32+26, (w0 >> 26) & 0x1, nb,parm);\
- OUT(op,i*32+27, (w0 >> 27) & 0x1, nb,parm);\
- OUT(op,i*32+28, (w0 >> 28) & 0x1, nb,parm);\
- OUT(op,i*32+29, (w0 >> 29) & 0x1, nb,parm);\
- OUT(op,i*32+30, (w0 >> 30) & 0x1, nb,parm);\
- OUT(op,i*32+31, (w0 >> 31) , nb,parm);;\
-}
-
-#define BITUNPACK64_1(ip, op, nb,parm) { \
- BITUNBLK64_1(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 1*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_2(ip, i, op, nb,parm) { uint64_t w0; w0 = *(uint64_t *)(ip+(i*1+0)*8/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x3, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 2) & 0x3, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 4) & 0x3, nb,parm);\
- OUT(op,i*32+ 3, (w0 >> 6) & 0x3, nb,parm);\
- OUT(op,i*32+ 4, (w0 >> 8) & 0x3, nb,parm);\
- OUT(op,i*32+ 5, (w0 >> 10) & 0x3, nb,parm);\
- OUT(op,i*32+ 6, (w0 >> 12) & 0x3, nb,parm);\
- OUT(op,i*32+ 7, (w0 >> 14) & 0x3, nb,parm);\
- OUT(op,i*32+ 8, (w0 >> 16) & 0x3, nb,parm);\
- OUT(op,i*32+ 9, (w0 >> 18) & 0x3, nb,parm);\
- OUT(op,i*32+10, (w0 >> 20) & 0x3, nb,parm);\
- OUT(op,i*32+11, (w0 >> 22) & 0x3, nb,parm);\
- OUT(op,i*32+12, (w0 >> 24) & 0x3, nb,parm);\
- OUT(op,i*32+13, (w0 >> 26) & 0x3, nb,parm);\
- OUT(op,i*32+14, (w0 >> 28) & 0x3, nb,parm);\
- OUT(op,i*32+15, (w0 >> 30) & 0x3, nb,parm);\
- OUT(op,i*32+16, (w0 >> 32) & 0x3, nb,parm);\
- OUT(op,i*32+17, (w0 >> 34) & 0x3, nb,parm);\
- OUT(op,i*32+18, (w0 >> 36) & 0x3, nb,parm);\
- OUT(op,i*32+19, (w0 >> 38) & 0x3, nb,parm);\
- OUT(op,i*32+20, (w0 >> 40) & 0x3, nb,parm);\
- OUT(op,i*32+21, (w0 >> 42) & 0x3, nb,parm);\
- OUT(op,i*32+22, (w0 >> 44) & 0x3, nb,parm);\
- OUT(op,i*32+23, (w0 >> 46) & 0x3, nb,parm);\
- OUT(op,i*32+24, (w0 >> 48) & 0x3, nb,parm);\
- OUT(op,i*32+25, (w0 >> 50) & 0x3, nb,parm);\
- OUT(op,i*32+26, (w0 >> 52) & 0x3, nb,parm);\
- OUT(op,i*32+27, (w0 >> 54) & 0x3, nb,parm);\
- OUT(op,i*32+28, (w0 >> 56) & 0x3, nb,parm);\
- OUT(op,i*32+29, (w0 >> 58) & 0x3, nb,parm);\
- OUT(op,i*32+30, (w0 >> 60) & 0x3, nb,parm);\
- OUT(op,i*32+31, (w0 >> 62) , nb,parm);;\
-}
-
-#define BITUNPACK64_2(ip, op, nb,parm) { \
- BITUNBLK64_2(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 2*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_3(ip, i, op, nb,parm) { uint64_t w0,w1; w0 = *(uint64_t *)(ip+(i*3+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x7, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 3) & 0x7, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 6) & 0x7, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 9) & 0x7, nb,parm);\
- OUT(op,i*64+ 4, (w0 >> 12) & 0x7, nb,parm);\
- OUT(op,i*64+ 5, (w0 >> 15) & 0x7, nb,parm);\
- OUT(op,i*64+ 6, (w0 >> 18) & 0x7, nb,parm);\
- OUT(op,i*64+ 7, (w0 >> 21) & 0x7, nb,parm);\
- OUT(op,i*64+ 8, (w0 >> 24) & 0x7, nb,parm);\
- OUT(op,i*64+ 9, (w0 >> 27) & 0x7, nb,parm);\
- OUT(op,i*64+10, (w0 >> 30) & 0x7, nb,parm);\
- OUT(op,i*64+11, (w0 >> 33) & 0x7, nb,parm);\
- OUT(op,i*64+12, (w0 >> 36) & 0x7, nb,parm);\
- OUT(op,i*64+13, (w0 >> 39) & 0x7, nb,parm);\
- OUT(op,i*64+14, (w0 >> 42) & 0x7, nb,parm);\
- OUT(op,i*64+15, (w0 >> 45) & 0x7, nb,parm);\
- OUT(op,i*64+16, (w0 >> 48) & 0x7, nb,parm);\
- OUT(op,i*64+17, (w0 >> 51) & 0x7, nb,parm);\
- OUT(op,i*64+18, (w0 >> 54) & 0x7, nb,parm);\
- OUT(op,i*64+19, (w0 >> 57) & 0x7, nb,parm);\
- OUT(op,i*64+20, (w0 >> 60) & 0x7, nb,parm); w1 = *(uint32_t *)(ip+(i*3+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+21, (w0 >> 63) | (w1 << 1) & 0x7, nb,parm);\
- OUT(op,i*64+22, (w1 >> 2) & 0x7, nb,parm);\
- OUT(op,i*64+23, (w1 >> 5) & 0x7, nb,parm);\
- OUT(op,i*64+24, (w1 >> 8) & 0x7, nb,parm);\
- OUT(op,i*64+25, (w1 >> 11) & 0x7, nb,parm);\
- OUT(op,i*64+26, (w1 >> 14) & 0x7, nb,parm);\
- OUT(op,i*64+27, (w1 >> 17) & 0x7, nb,parm);\
- OUT(op,i*64+28, (w1 >> 20) & 0x7, nb,parm);\
- OUT(op,i*64+29, (w1 >> 23) & 0x7, nb,parm);\
- OUT(op,i*64+30, (w1 >> 26) & 0x7, nb,parm);\
- OUT(op,i*64+31, (w1 >> 29) & 0x7, nb,parm);;\
-}
-
-#define BITUNPACK64_3(ip, op, nb,parm) { \
- BITUNBLK64_3(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 3*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_4(ip, i, op, nb,parm) { uint64_t w0,w1; w0 = *(uint64_t *)(ip+(i*1+0)*8/sizeof(ip[0]));\
- OUT(op,i*16+ 0, (w0 ) & 0xf, nb,parm);\
- OUT(op,i*16+ 1, (w0 >> 4) & 0xf, nb,parm);\
- OUT(op,i*16+ 2, (w0 >> 8) & 0xf, nb,parm);\
- OUT(op,i*16+ 3, (w0 >> 12) & 0xf, nb,parm);\
- OUT(op,i*16+ 4, (w0 >> 16) & 0xf, nb,parm);\
- OUT(op,i*16+ 5, (w0 >> 20) & 0xf, nb,parm);\
- OUT(op,i*16+ 6, (w0 >> 24) & 0xf, nb,parm);\
- OUT(op,i*16+ 7, (w0 >> 28) & 0xf, nb,parm);\
- OUT(op,i*16+ 8, (w0 >> 32) & 0xf, nb,parm);\
- OUT(op,i*16+ 9, (w0 >> 36) & 0xf, nb,parm);\
- OUT(op,i*16+10, (w0 >> 40) & 0xf, nb,parm);\
- OUT(op,i*16+11, (w0 >> 44) & 0xf, nb,parm);\
- OUT(op,i*16+12, (w0 >> 48) & 0xf, nb,parm);\
- OUT(op,i*16+13, (w0 >> 52) & 0xf, nb,parm);\
- OUT(op,i*16+14, (w0 >> 56) & 0xf, nb,parm);\
- OUT(op,i*16+15, (w0 >> 60) , nb,parm);;\
-}
-
-#define BITUNPACK64_4(ip, op, nb,parm) { \
- BITUNBLK64_4(ip, 0, op, nb,parm);\
- BITUNBLK64_4(ip, 1, op, nb,parm); OPI(op, nb,parm); ip += 4*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_5(ip, i, op, nb,parm) { uint64_t w0,w1,w2; w0 = *(uint64_t *)(ip+(i*5+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x1f, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 5) & 0x1f, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 10) & 0x1f, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 15) & 0x1f, nb,parm);\
- OUT(op,i*64+ 4, (w0 >> 20) & 0x1f, nb,parm);\
- OUT(op,i*64+ 5, (w0 >> 25) & 0x1f, nb,parm);\
- OUT(op,i*64+ 6, (w0 >> 30) & 0x1f, nb,parm);\
- OUT(op,i*64+ 7, (w0 >> 35) & 0x1f, nb,parm);\
- OUT(op,i*64+ 8, (w0 >> 40) & 0x1f, nb,parm);\
- OUT(op,i*64+ 9, (w0 >> 45) & 0x1f, nb,parm);\
- OUT(op,i*64+10, (w0 >> 50) & 0x1f, nb,parm);\
- OUT(op,i*64+11, (w0 >> 55) & 0x1f, nb,parm); w1 = *(uint64_t *)(ip+(i*5+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+12, (w0 >> 60) | (w1 << 4) & 0x1f, nb,parm);\
- OUT(op,i*64+13, (w1 >> 1) & 0x1f, nb,parm);\
- OUT(op,i*64+14, (w1 >> 6) & 0x1f, nb,parm);\
- OUT(op,i*64+15, (w1 >> 11) & 0x1f, nb,parm);\
- OUT(op,i*64+16, (w1 >> 16) & 0x1f, nb,parm);\
- OUT(op,i*64+17, (w1 >> 21) & 0x1f, nb,parm);\
- OUT(op,i*64+18, (w1 >> 26) & 0x1f, nb,parm);\
- OUT(op,i*64+19, (w1 >> 31) & 0x1f, nb,parm);\
- OUT(op,i*64+20, (w1 >> 36) & 0x1f, nb,parm);\
- OUT(op,i*64+21, (w1 >> 41) & 0x1f, nb,parm);\
- OUT(op,i*64+22, (w1 >> 46) & 0x1f, nb,parm);\
- OUT(op,i*64+23, (w1 >> 51) & 0x1f, nb,parm);\
- OUT(op,i*64+24, (w1 >> 56) & 0x1f, nb,parm); w2 = *(uint32_t *)(ip+(i*5+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+25, (w1 >> 61) | (w2 << 3) & 0x1f, nb,parm);\
- OUT(op,i*64+26, (w2 >> 2) & 0x1f, nb,parm);\
- OUT(op,i*64+27, (w2 >> 7) & 0x1f, nb,parm);\
- OUT(op,i*64+28, (w2 >> 12) & 0x1f, nb,parm);\
- OUT(op,i*64+29, (w2 >> 17) & 0x1f, nb,parm);\
- OUT(op,i*64+30, (w2 >> 22) & 0x1f, nb,parm);\
- OUT(op,i*64+31, (w2 >> 27) & 0x1f, nb,parm);;\
-}
-
-#define BITUNPACK64_5(ip, op, nb,parm) { \
- BITUNBLK64_5(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 5*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_6(ip, i, op, nb,parm) { uint64_t w0,w1,w2; w0 = *(uint64_t *)(ip+(i*3+0)*8/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x3f, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 6) & 0x3f, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 12) & 0x3f, nb,parm);\
- OUT(op,i*32+ 3, (w0 >> 18) & 0x3f, nb,parm);\
- OUT(op,i*32+ 4, (w0 >> 24) & 0x3f, nb,parm);\
- OUT(op,i*32+ 5, (w0 >> 30) & 0x3f, nb,parm);\
- OUT(op,i*32+ 6, (w0 >> 36) & 0x3f, nb,parm);\
- OUT(op,i*32+ 7, (w0 >> 42) & 0x3f, nb,parm);\
- OUT(op,i*32+ 8, (w0 >> 48) & 0x3f, nb,parm);\
- OUT(op,i*32+ 9, (w0 >> 54) & 0x3f, nb,parm); w1 = *(uint64_t *)(ip+(i*3+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+10, (w0 >> 60) | (w1 << 4) & 0x3f, nb,parm);\
- OUT(op,i*32+11, (w1 >> 2) & 0x3f, nb,parm);\
- OUT(op,i*32+12, (w1 >> 8) & 0x3f, nb,parm);\
- OUT(op,i*32+13, (w1 >> 14) & 0x3f, nb,parm);\
- OUT(op,i*32+14, (w1 >> 20) & 0x3f, nb,parm);\
- OUT(op,i*32+15, (w1 >> 26) & 0x3f, nb,parm);\
- OUT(op,i*32+16, (w1 >> 32) & 0x3f, nb,parm);\
- OUT(op,i*32+17, (w1 >> 38) & 0x3f, nb,parm);\
- OUT(op,i*32+18, (w1 >> 44) & 0x3f, nb,parm);\
- OUT(op,i*32+19, (w1 >> 50) & 0x3f, nb,parm);\
- OUT(op,i*32+20, (w1 >> 56) & 0x3f, nb,parm); w2 = *(uint64_t *)(ip+(i*3+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+21, (w1 >> 62) | (w2 << 2) & 0x3f, nb,parm);\
- OUT(op,i*32+22, (w2 >> 4) & 0x3f, nb,parm);\
- OUT(op,i*32+23, (w2 >> 10) & 0x3f, nb,parm);\
- OUT(op,i*32+24, (w2 >> 16) & 0x3f, nb,parm);\
- OUT(op,i*32+25, (w2 >> 22) & 0x3f, nb,parm);\
- OUT(op,i*32+26, (w2 >> 28) & 0x3f, nb,parm);\
- OUT(op,i*32+27, (w2 >> 34) & 0x3f, nb,parm);\
- OUT(op,i*32+28, (w2 >> 40) & 0x3f, nb,parm);\
- OUT(op,i*32+29, (w2 >> 46) & 0x3f, nb,parm);\
- OUT(op,i*32+30, (w2 >> 52) & 0x3f, nb,parm);\
- OUT(op,i*32+31, (w2 >> 58) , nb,parm);;\
-}
-
-#define BITUNPACK64_6(ip, op, nb,parm) { \
- BITUNBLK64_6(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 6*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_7(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3; w0 = *(uint64_t *)(ip+(i*7+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x7f, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 7) & 0x7f, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 14) & 0x7f, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 21) & 0x7f, nb,parm);\
- OUT(op,i*64+ 4, (w0 >> 28) & 0x7f, nb,parm);\
- OUT(op,i*64+ 5, (w0 >> 35) & 0x7f, nb,parm);\
- OUT(op,i*64+ 6, (w0 >> 42) & 0x7f, nb,parm);\
- OUT(op,i*64+ 7, (w0 >> 49) & 0x7f, nb,parm);\
- OUT(op,i*64+ 8, (w0 >> 56) & 0x7f, nb,parm); w1 = *(uint64_t *)(ip+(i*7+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 9, (w0 >> 63) | (w1 << 1) & 0x7f, nb,parm);\
- OUT(op,i*64+10, (w1 >> 6) & 0x7f, nb,parm);\
- OUT(op,i*64+11, (w1 >> 13) & 0x7f, nb,parm);\
- OUT(op,i*64+12, (w1 >> 20) & 0x7f, nb,parm);\
- OUT(op,i*64+13, (w1 >> 27) & 0x7f, nb,parm);\
- OUT(op,i*64+14, (w1 >> 34) & 0x7f, nb,parm);\
- OUT(op,i*64+15, (w1 >> 41) & 0x7f, nb,parm);\
- OUT(op,i*64+16, (w1 >> 48) & 0x7f, nb,parm);\
- OUT(op,i*64+17, (w1 >> 55) & 0x7f, nb,parm); w2 = *(uint64_t *)(ip+(i*7+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+18, (w1 >> 62) | (w2 << 2) & 0x7f, nb,parm);\
- OUT(op,i*64+19, (w2 >> 5) & 0x7f, nb,parm);\
- OUT(op,i*64+20, (w2 >> 12) & 0x7f, nb,parm);\
- OUT(op,i*64+21, (w2 >> 19) & 0x7f, nb,parm);\
- OUT(op,i*64+22, (w2 >> 26) & 0x7f, nb,parm);\
- OUT(op,i*64+23, (w2 >> 33) & 0x7f, nb,parm);\
- OUT(op,i*64+24, (w2 >> 40) & 0x7f, nb,parm);\
- OUT(op,i*64+25, (w2 >> 47) & 0x7f, nb,parm);\
- OUT(op,i*64+26, (w2 >> 54) & 0x7f, nb,parm); w3 = *(uint32_t *)(ip+(i*7+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+27, (w2 >> 61) | (w3 << 3) & 0x7f, nb,parm);\
- OUT(op,i*64+28, (w3 >> 4) & 0x7f, nb,parm);\
- OUT(op,i*64+29, (w3 >> 11) & 0x7f, nb,parm);\
- OUT(op,i*64+30, (w3 >> 18) & 0x7f, nb,parm);\
- OUT(op,i*64+31, (w3 >> 25) & 0x7f, nb,parm);;\
-}
-
-#define BITUNPACK64_7(ip, op, nb,parm) { \
- BITUNBLK64_7(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 7*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_8(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3; w0 = *(uint64_t *)(ip+(i*1+0)*8/sizeof(ip[0]));\
- OUT(op,i*8+ 0, (w0 ) & 0xff, nb,parm);\
- OUT(op,i*8+ 1, (w0 >> 8) & 0xff, nb,parm);\
- OUT(op,i*8+ 2, (w0 >> 16) & 0xff, nb,parm);\
- OUT(op,i*8+ 3, (w0 >> 24) & 0xff, nb,parm);\
- OUT(op,i*8+ 4, (w0 >> 32) & 0xff, nb,parm);\
- OUT(op,i*8+ 5, (w0 >> 40) & 0xff, nb,parm);\
- OUT(op,i*8+ 6, (w0 >> 48) & 0xff, nb,parm);\
- OUT(op,i*8+ 7, (w0 >> 56) , nb,parm);;\
-}
-
-#define BITUNPACK64_8(ip, op, nb,parm) { \
- BITUNBLK64_8(ip, 0, op, nb,parm);\
- BITUNBLK64_8(ip, 1, op, nb,parm);\
- BITUNBLK64_8(ip, 2, op, nb,parm);\
- BITUNBLK64_8(ip, 3, op, nb,parm); OPI(op, nb,parm); ip += 8*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_9(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4; w0 = *(uint64_t *)(ip+(i*9+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 9) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 18) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 27) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 4, (w0 >> 36) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 5, (w0 >> 45) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 6, (w0 >> 54) & 0x1ff, nb,parm); w1 = *(uint64_t *)(ip+(i*9+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 7, (w0 >> 63) | (w1 << 1) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 8, (w1 >> 8) & 0x1ff, nb,parm);\
- OUT(op,i*64+ 9, (w1 >> 17) & 0x1ff, nb,parm);\
- OUT(op,i*64+10, (w1 >> 26) & 0x1ff, nb,parm);\
- OUT(op,i*64+11, (w1 >> 35) & 0x1ff, nb,parm);\
- OUT(op,i*64+12, (w1 >> 44) & 0x1ff, nb,parm);\
- OUT(op,i*64+13, (w1 >> 53) & 0x1ff, nb,parm); w2 = *(uint64_t *)(ip+(i*9+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+14, (w1 >> 62) | (w2 << 2) & 0x1ff, nb,parm);\
- OUT(op,i*64+15, (w2 >> 7) & 0x1ff, nb,parm);\
- OUT(op,i*64+16, (w2 >> 16) & 0x1ff, nb,parm);\
- OUT(op,i*64+17, (w2 >> 25) & 0x1ff, nb,parm);\
- OUT(op,i*64+18, (w2 >> 34) & 0x1ff, nb,parm);\
- OUT(op,i*64+19, (w2 >> 43) & 0x1ff, nb,parm);\
- OUT(op,i*64+20, (w2 >> 52) & 0x1ff, nb,parm); w3 = *(uint64_t *)(ip+(i*9+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+21, (w2 >> 61) | (w3 << 3) & 0x1ff, nb,parm);\
- OUT(op,i*64+22, (w3 >> 6) & 0x1ff, nb,parm);\
- OUT(op,i*64+23, (w3 >> 15) & 0x1ff, nb,parm);\
- OUT(op,i*64+24, (w3 >> 24) & 0x1ff, nb,parm);\
- OUT(op,i*64+25, (w3 >> 33) & 0x1ff, nb,parm);\
- OUT(op,i*64+26, (w3 >> 42) & 0x1ff, nb,parm);\
- OUT(op,i*64+27, (w3 >> 51) & 0x1ff, nb,parm); w4 = *(uint32_t *)(ip+(i*9+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+28, (w3 >> 60) | (w4 << 4) & 0x1ff, nb,parm);\
- OUT(op,i*64+29, (w4 >> 5) & 0x1ff, nb,parm);\
- OUT(op,i*64+30, (w4 >> 14) & 0x1ff, nb,parm);\
- OUT(op,i*64+31, (w4 >> 23) & 0x1ff, nb,parm);;\
-}
-
-#define BITUNPACK64_9(ip, op, nb,parm) { \
- BITUNBLK64_9(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 9*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_10(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4; w0 = *(uint64_t *)(ip+(i*5+0)*8/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 10) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 20) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 3, (w0 >> 30) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 4, (w0 >> 40) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 5, (w0 >> 50) & 0x3ff, nb,parm); w1 = *(uint64_t *)(ip+(i*5+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+ 6, (w0 >> 60) | (w1 << 4) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 7, (w1 >> 6) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 8, (w1 >> 16) & 0x3ff, nb,parm);\
- OUT(op,i*32+ 9, (w1 >> 26) & 0x3ff, nb,parm);\
- OUT(op,i*32+10, (w1 >> 36) & 0x3ff, nb,parm);\
- OUT(op,i*32+11, (w1 >> 46) & 0x3ff, nb,parm); w2 = *(uint64_t *)(ip+(i*5+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+12, (w1 >> 56) | (w2 << 8) & 0x3ff, nb,parm);\
- OUT(op,i*32+13, (w2 >> 2) & 0x3ff, nb,parm);\
- OUT(op,i*32+14, (w2 >> 12) & 0x3ff, nb,parm);\
- OUT(op,i*32+15, (w2 >> 22) & 0x3ff, nb,parm);\
- OUT(op,i*32+16, (w2 >> 32) & 0x3ff, nb,parm);\
- OUT(op,i*32+17, (w2 >> 42) & 0x3ff, nb,parm);\
- OUT(op,i*32+18, (w2 >> 52) & 0x3ff, nb,parm); w3 = *(uint64_t *)(ip+(i*5+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+19, (w2 >> 62) | (w3 << 2) & 0x3ff, nb,parm);\
- OUT(op,i*32+20, (w3 >> 8) & 0x3ff, nb,parm);\
- OUT(op,i*32+21, (w3 >> 18) & 0x3ff, nb,parm);\
- OUT(op,i*32+22, (w3 >> 28) & 0x3ff, nb,parm);\
- OUT(op,i*32+23, (w3 >> 38) & 0x3ff, nb,parm);\
- OUT(op,i*32+24, (w3 >> 48) & 0x3ff, nb,parm); w4 = *(uint64_t *)(ip+(i*5+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+25, (w3 >> 58) | (w4 << 6) & 0x3ff, nb,parm);\
- OUT(op,i*32+26, (w4 >> 4) & 0x3ff, nb,parm);\
- OUT(op,i*32+27, (w4 >> 14) & 0x3ff, nb,parm);\
- OUT(op,i*32+28, (w4 >> 24) & 0x3ff, nb,parm);\
- OUT(op,i*32+29, (w4 >> 34) & 0x3ff, nb,parm);\
- OUT(op,i*32+30, (w4 >> 44) & 0x3ff, nb,parm);\
- OUT(op,i*32+31, (w4 >> 54) , nb,parm);;\
-}
-
-#define BITUNPACK64_10(ip, op, nb,parm) { \
- BITUNBLK64_10(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 10*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_11(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5; w0 = *(uint64_t *)(ip+(i*11+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 11) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 22) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 33) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 4, (w0 >> 44) & 0x7ff, nb,parm); w1 = *(uint64_t *)(ip+(i*11+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 5, (w0 >> 55) | (w1 << 9) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 6, (w1 >> 2) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 7, (w1 >> 13) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 8, (w1 >> 24) & 0x7ff, nb,parm);\
- OUT(op,i*64+ 9, (w1 >> 35) & 0x7ff, nb,parm);\
- OUT(op,i*64+10, (w1 >> 46) & 0x7ff, nb,parm); w2 = *(uint64_t *)(ip+(i*11+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+11, (w1 >> 57) | (w2 << 7) & 0x7ff, nb,parm);\
- OUT(op,i*64+12, (w2 >> 4) & 0x7ff, nb,parm);\
- OUT(op,i*64+13, (w2 >> 15) & 0x7ff, nb,parm);\
- OUT(op,i*64+14, (w2 >> 26) & 0x7ff, nb,parm);\
- OUT(op,i*64+15, (w2 >> 37) & 0x7ff, nb,parm);\
- OUT(op,i*64+16, (w2 >> 48) & 0x7ff, nb,parm); w3 = *(uint64_t *)(ip+(i*11+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+17, (w2 >> 59) | (w3 << 5) & 0x7ff, nb,parm);\
- OUT(op,i*64+18, (w3 >> 6) & 0x7ff, nb,parm);\
- OUT(op,i*64+19, (w3 >> 17) & 0x7ff, nb,parm);\
- OUT(op,i*64+20, (w3 >> 28) & 0x7ff, nb,parm);\
- OUT(op,i*64+21, (w3 >> 39) & 0x7ff, nb,parm);\
- OUT(op,i*64+22, (w3 >> 50) & 0x7ff, nb,parm); w4 = *(uint64_t *)(ip+(i*11+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+23, (w3 >> 61) | (w4 << 3) & 0x7ff, nb,parm);\
- OUT(op,i*64+24, (w4 >> 8) & 0x7ff, nb,parm);\
- OUT(op,i*64+25, (w4 >> 19) & 0x7ff, nb,parm);\
- OUT(op,i*64+26, (w4 >> 30) & 0x7ff, nb,parm);\
- OUT(op,i*64+27, (w4 >> 41) & 0x7ff, nb,parm);\
- OUT(op,i*64+28, (w4 >> 52) & 0x7ff, nb,parm); w5 = *(uint32_t *)(ip+(i*11+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+29, (w4 >> 63) | (w5 << 1) & 0x7ff, nb,parm);\
- OUT(op,i*64+30, (w5 >> 10) & 0x7ff, nb,parm);\
- OUT(op,i*64+31, (w5 >> 21) & 0x7ff, nb,parm);;\
-}
-
-#define BITUNPACK64_11(ip, op, nb,parm) { \
- BITUNBLK64_11(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 11*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_12(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5; w0 = *(uint64_t *)(ip+(i*3+0)*8/sizeof(ip[0]));\
- OUT(op,i*16+ 0, (w0 ) & 0xfff, nb,parm);\
- OUT(op,i*16+ 1, (w0 >> 12) & 0xfff, nb,parm);\
- OUT(op,i*16+ 2, (w0 >> 24) & 0xfff, nb,parm);\
- OUT(op,i*16+ 3, (w0 >> 36) & 0xfff, nb,parm);\
- OUT(op,i*16+ 4, (w0 >> 48) & 0xfff, nb,parm); w1 = *(uint64_t *)(ip+(i*3+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+ 5, (w0 >> 60) | (w1 << 4) & 0xfff, nb,parm);\
- OUT(op,i*16+ 6, (w1 >> 8) & 0xfff, nb,parm);\
- OUT(op,i*16+ 7, (w1 >> 20) & 0xfff, nb,parm);\
- OUT(op,i*16+ 8, (w1 >> 32) & 0xfff, nb,parm);\
- OUT(op,i*16+ 9, (w1 >> 44) & 0xfff, nb,parm); w2 = *(uint64_t *)(ip+(i*3+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+10, (w1 >> 56) | (w2 << 8) & 0xfff, nb,parm);\
- OUT(op,i*16+11, (w2 >> 4) & 0xfff, nb,parm);\
- OUT(op,i*16+12, (w2 >> 16) & 0xfff, nb,parm);\
- OUT(op,i*16+13, (w2 >> 28) & 0xfff, nb,parm);\
- OUT(op,i*16+14, (w2 >> 40) & 0xfff, nb,parm);\
- OUT(op,i*16+15, (w2 >> 52) , nb,parm);;\
-}
-
-#define BITUNPACK64_12(ip, op, nb,parm) { \
- BITUNBLK64_12(ip, 0, op, nb,parm);\
- BITUNBLK64_12(ip, 1, op, nb,parm); OPI(op, nb,parm); ip += 12*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_13(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6; w0 = *(uint64_t *)(ip+(i*13+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 13) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 26) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 39) & 0x1fff, nb,parm); w1 = *(uint64_t *)(ip+(i*13+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 4, (w0 >> 52) | (w1 << 12) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 5, (w1 >> 1) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 6, (w1 >> 14) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 7, (w1 >> 27) & 0x1fff, nb,parm);\
- OUT(op,i*64+ 8, (w1 >> 40) & 0x1fff, nb,parm); w2 = *(uint64_t *)(ip+(i*13+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 9, (w1 >> 53) | (w2 << 11) & 0x1fff, nb,parm);\
- OUT(op,i*64+10, (w2 >> 2) & 0x1fff, nb,parm);\
- OUT(op,i*64+11, (w2 >> 15) & 0x1fff, nb,parm);\
- OUT(op,i*64+12, (w2 >> 28) & 0x1fff, nb,parm);\
- OUT(op,i*64+13, (w2 >> 41) & 0x1fff, nb,parm); w3 = *(uint64_t *)(ip+(i*13+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+14, (w2 >> 54) | (w3 << 10) & 0x1fff, nb,parm);\
- OUT(op,i*64+15, (w3 >> 3) & 0x1fff, nb,parm);\
- OUT(op,i*64+16, (w3 >> 16) & 0x1fff, nb,parm);\
- OUT(op,i*64+17, (w3 >> 29) & 0x1fff, nb,parm);\
- OUT(op,i*64+18, (w3 >> 42) & 0x1fff, nb,parm); w4 = *(uint64_t *)(ip+(i*13+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+19, (w3 >> 55) | (w4 << 9) & 0x1fff, nb,parm);\
- OUT(op,i*64+20, (w4 >> 4) & 0x1fff, nb,parm);\
- OUT(op,i*64+21, (w4 >> 17) & 0x1fff, nb,parm);\
- OUT(op,i*64+22, (w4 >> 30) & 0x1fff, nb,parm);\
- OUT(op,i*64+23, (w4 >> 43) & 0x1fff, nb,parm); w5 = *(uint64_t *)(ip+(i*13+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+24, (w4 >> 56) | (w5 << 8) & 0x1fff, nb,parm);\
- OUT(op,i*64+25, (w5 >> 5) & 0x1fff, nb,parm);\
- OUT(op,i*64+26, (w5 >> 18) & 0x1fff, nb,parm);\
- OUT(op,i*64+27, (w5 >> 31) & 0x1fff, nb,parm);\
- OUT(op,i*64+28, (w5 >> 44) & 0x1fff, nb,parm); w6 = *(uint32_t *)(ip+(i*13+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+29, (w5 >> 57) | (w6 << 7) & 0x1fff, nb,parm);\
- OUT(op,i*64+30, (w6 >> 6) & 0x1fff, nb,parm);\
- OUT(op,i*64+31, (w6 >> 19) & 0x1fff, nb,parm);;\
-}
-
-#define BITUNPACK64_13(ip, op, nb,parm) { \
- BITUNBLK64_13(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 13*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_14(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6; w0 = *(uint64_t *)(ip+(i*7+0)*8/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 14) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 28) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 3, (w0 >> 42) & 0x3fff, nb,parm); w1 = *(uint64_t *)(ip+(i*7+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+ 4, (w0 >> 56) | (w1 << 8) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 5, (w1 >> 6) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 6, (w1 >> 20) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 7, (w1 >> 34) & 0x3fff, nb,parm);\
- OUT(op,i*32+ 8, (w1 >> 48) & 0x3fff, nb,parm); w2 = *(uint64_t *)(ip+(i*7+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+ 9, (w1 >> 62) | (w2 << 2) & 0x3fff, nb,parm);\
- OUT(op,i*32+10, (w2 >> 12) & 0x3fff, nb,parm);\
- OUT(op,i*32+11, (w2 >> 26) & 0x3fff, nb,parm);\
- OUT(op,i*32+12, (w2 >> 40) & 0x3fff, nb,parm); w3 = *(uint64_t *)(ip+(i*7+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+13, (w2 >> 54) | (w3 << 10) & 0x3fff, nb,parm);\
- OUT(op,i*32+14, (w3 >> 4) & 0x3fff, nb,parm);\
- OUT(op,i*32+15, (w3 >> 18) & 0x3fff, nb,parm);\
- OUT(op,i*32+16, (w3 >> 32) & 0x3fff, nb,parm);\
- OUT(op,i*32+17, (w3 >> 46) & 0x3fff, nb,parm); w4 = *(uint64_t *)(ip+(i*7+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+18, (w3 >> 60) | (w4 << 4) & 0x3fff, nb,parm);\
- OUT(op,i*32+19, (w4 >> 10) & 0x3fff, nb,parm);\
- OUT(op,i*32+20, (w4 >> 24) & 0x3fff, nb,parm);\
- OUT(op,i*32+21, (w4 >> 38) & 0x3fff, nb,parm); w5 = *(uint64_t *)(ip+(i*7+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+22, (w4 >> 52) | (w5 << 12) & 0x3fff, nb,parm);\
- OUT(op,i*32+23, (w5 >> 2) & 0x3fff, nb,parm);\
- OUT(op,i*32+24, (w5 >> 16) & 0x3fff, nb,parm);\
- OUT(op,i*32+25, (w5 >> 30) & 0x3fff, nb,parm);\
- OUT(op,i*32+26, (w5 >> 44) & 0x3fff, nb,parm); w6 = *(uint64_t *)(ip+(i*7+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+27, (w5 >> 58) | (w6 << 6) & 0x3fff, nb,parm);\
- OUT(op,i*32+28, (w6 >> 8) & 0x3fff, nb,parm);\
- OUT(op,i*32+29, (w6 >> 22) & 0x3fff, nb,parm);\
- OUT(op,i*32+30, (w6 >> 36) & 0x3fff, nb,parm);\
- OUT(op,i*32+31, (w6 >> 50) , nb,parm);;\
-}
-
-#define BITUNPACK64_14(ip, op, nb,parm) { \
- BITUNBLK64_14(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 14*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_15(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7; w0 = *(uint64_t *)(ip+(i*15+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 15) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 30) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 3, (w0 >> 45) & 0x7fff, nb,parm); w1 = *(uint64_t *)(ip+(i*15+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 4, (w0 >> 60) | (w1 << 4) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 5, (w1 >> 11) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 6, (w1 >> 26) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 7, (w1 >> 41) & 0x7fff, nb,parm); w2 = *(uint64_t *)(ip+(i*15+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 8, (w1 >> 56) | (w2 << 8) & 0x7fff, nb,parm);\
- OUT(op,i*64+ 9, (w2 >> 7) & 0x7fff, nb,parm);\
- OUT(op,i*64+10, (w2 >> 22) & 0x7fff, nb,parm);\
- OUT(op,i*64+11, (w2 >> 37) & 0x7fff, nb,parm); w3 = *(uint64_t *)(ip+(i*15+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+12, (w2 >> 52) | (w3 << 12) & 0x7fff, nb,parm);\
- OUT(op,i*64+13, (w3 >> 3) & 0x7fff, nb,parm);\
- OUT(op,i*64+14, (w3 >> 18) & 0x7fff, nb,parm);\
- OUT(op,i*64+15, (w3 >> 33) & 0x7fff, nb,parm);\
- OUT(op,i*64+16, (w3 >> 48) & 0x7fff, nb,parm); w4 = *(uint64_t *)(ip+(i*15+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+17, (w3 >> 63) | (w4 << 1) & 0x7fff, nb,parm);\
- OUT(op,i*64+18, (w4 >> 14) & 0x7fff, nb,parm);\
- OUT(op,i*64+19, (w4 >> 29) & 0x7fff, nb,parm);\
- OUT(op,i*64+20, (w4 >> 44) & 0x7fff, nb,parm); w5 = *(uint64_t *)(ip+(i*15+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+21, (w4 >> 59) | (w5 << 5) & 0x7fff, nb,parm);\
- OUT(op,i*64+22, (w5 >> 10) & 0x7fff, nb,parm);\
- OUT(op,i*64+23, (w5 >> 25) & 0x7fff, nb,parm);\
- OUT(op,i*64+24, (w5 >> 40) & 0x7fff, nb,parm); w6 = *(uint64_t *)(ip+(i*15+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+25, (w5 >> 55) | (w6 << 9) & 0x7fff, nb,parm);\
- OUT(op,i*64+26, (w6 >> 6) & 0x7fff, nb,parm);\
- OUT(op,i*64+27, (w6 >> 21) & 0x7fff, nb,parm);\
- OUT(op,i*64+28, (w6 >> 36) & 0x7fff, nb,parm); w7 = *(uint32_t *)(ip+(i*15+7)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+29, (w6 >> 51) | (w7 << 13) & 0x7fff, nb,parm);\
- OUT(op,i*64+30, (w7 >> 2) & 0x7fff, nb,parm);\
- OUT(op,i*64+31, (w7 >> 17) & 0x7fff, nb,parm);;\
-}
-
-#define BITUNPACK64_15(ip, op, nb,parm) { \
- BITUNBLK64_15(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 15*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_16(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7;\
- OUT(op,i*4+ 0, *(uint16_t *)(ip+i*8+ 0), nb,parm);\
- OUT(op,i*4+ 1, *(uint16_t *)(ip+i*8+ 2), nb,parm);\
- OUT(op,i*4+ 2, *(uint16_t *)(ip+i*8+ 4), nb,parm);\
- OUT(op,i*4+ 3, *(uint16_t *)(ip+i*8+ 6), nb,parm);;\
-}
-
-#define BITUNPACK64_16(ip, op, nb,parm) { \
- BITUNBLK64_16(ip, 0, op, nb,parm);\
- BITUNBLK64_16(ip, 1, op, nb,parm);\
- BITUNBLK64_16(ip, 2, op, nb,parm);\
- BITUNBLK64_16(ip, 3, op, nb,parm);\
- BITUNBLK64_16(ip, 4, op, nb,parm);\
- BITUNBLK64_16(ip, 5, op, nb,parm);\
- BITUNBLK64_16(ip, 6, op, nb,parm);\
- BITUNBLK64_16(ip, 7, op, nb,parm); OPI(op, nb,parm); ip += 16*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_17(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7,w8; w0 = *(uint64_t *)(ip+(i*17+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 17) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 34) & 0x1ffff, nb,parm); w1 = *(uint64_t *)(ip+(i*17+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 3, (w0 >> 51) | (w1 << 13) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 4, (w1 >> 4) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 5, (w1 >> 21) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 6, (w1 >> 38) & 0x1ffff, nb,parm); w2 = *(uint64_t *)(ip+(i*17+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 7, (w1 >> 55) | (w2 << 9) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 8, (w2 >> 8) & 0x1ffff, nb,parm);\
- OUT(op,i*64+ 9, (w2 >> 25) & 0x1ffff, nb,parm);\
- OUT(op,i*64+10, (w2 >> 42) & 0x1ffff, nb,parm); w3 = *(uint64_t *)(ip+(i*17+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+11, (w2 >> 59) | (w3 << 5) & 0x1ffff, nb,parm);\
- OUT(op,i*64+12, (w3 >> 12) & 0x1ffff, nb,parm);\
- OUT(op,i*64+13, (w3 >> 29) & 0x1ffff, nb,parm);\
- OUT(op,i*64+14, (w3 >> 46) & 0x1ffff, nb,parm); w4 = *(uint64_t *)(ip+(i*17+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+15, (w3 >> 63) | (w4 << 1) & 0x1ffff, nb,parm);\
- OUT(op,i*64+16, (w4 >> 16) & 0x1ffff, nb,parm);\
- OUT(op,i*64+17, (w4 >> 33) & 0x1ffff, nb,parm); w5 = *(uint64_t *)(ip+(i*17+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+18, (w4 >> 50) | (w5 << 14) & 0x1ffff, nb,parm);\
- OUT(op,i*64+19, (w5 >> 3) & 0x1ffff, nb,parm);\
- OUT(op,i*64+20, (w5 >> 20) & 0x1ffff, nb,parm);\
- OUT(op,i*64+21, (w5 >> 37) & 0x1ffff, nb,parm); w6 = *(uint64_t *)(ip+(i*17+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+22, (w5 >> 54) | (w6 << 10) & 0x1ffff, nb,parm);\
- OUT(op,i*64+23, (w6 >> 7) & 0x1ffff, nb,parm);\
- OUT(op,i*64+24, (w6 >> 24) & 0x1ffff, nb,parm);\
- OUT(op,i*64+25, (w6 >> 41) & 0x1ffff, nb,parm); w7 = *(uint64_t *)(ip+(i*17+7)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+26, (w6 >> 58) | (w7 << 6) & 0x1ffff, nb,parm);\
- OUT(op,i*64+27, (w7 >> 11) & 0x1ffff, nb,parm);\
- OUT(op,i*64+28, (w7 >> 28) & 0x1ffff, nb,parm);\
- OUT(op,i*64+29, (w7 >> 45) & 0x1ffff, nb,parm); w8 = *(uint32_t *)(ip+(i*17+8)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+30, (w7 >> 62) | (w8 << 2) & 0x1ffff, nb,parm);\
- OUT(op,i*64+31, (w8 >> 15) & 0x1ffff, nb,parm);;\
-}
-
-#define BITUNPACK64_17(ip, op, nb,parm) { \
- BITUNBLK64_17(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 17*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_18(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7,w8; w0 = *(uint64_t *)(ip+(i*9+0)*8/sizeof(ip[0]));\
- OUT(op,i*32+ 0, (w0 ) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 1, (w0 >> 18) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 2, (w0 >> 36) & 0x3ffff, nb,parm); w1 = *(uint64_t *)(ip+(i*9+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+ 3, (w0 >> 54) | (w1 << 10) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 4, (w1 >> 8) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 5, (w1 >> 26) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 6, (w1 >> 44) & 0x3ffff, nb,parm); w2 = *(uint64_t *)(ip+(i*9+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+ 7, (w1 >> 62) | (w2 << 2) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 8, (w2 >> 16) & 0x3ffff, nb,parm);\
- OUT(op,i*32+ 9, (w2 >> 34) & 0x3ffff, nb,parm); w3 = *(uint64_t *)(ip+(i*9+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+10, (w2 >> 52) | (w3 << 12) & 0x3ffff, nb,parm);\
- OUT(op,i*32+11, (w3 >> 6) & 0x3ffff, nb,parm);\
- OUT(op,i*32+12, (w3 >> 24) & 0x3ffff, nb,parm);\
- OUT(op,i*32+13, (w3 >> 42) & 0x3ffff, nb,parm); w4 = *(uint64_t *)(ip+(i*9+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+14, (w3 >> 60) | (w4 << 4) & 0x3ffff, nb,parm);\
- OUT(op,i*32+15, (w4 >> 14) & 0x3ffff, nb,parm);\
- OUT(op,i*32+16, (w4 >> 32) & 0x3ffff, nb,parm); w5 = *(uint64_t *)(ip+(i*9+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+17, (w4 >> 50) | (w5 << 14) & 0x3ffff, nb,parm);\
- OUT(op,i*32+18, (w5 >> 4) & 0x3ffff, nb,parm);\
- OUT(op,i*32+19, (w5 >> 22) & 0x3ffff, nb,parm);\
- OUT(op,i*32+20, (w5 >> 40) & 0x3ffff, nb,parm); w6 = *(uint64_t *)(ip+(i*9+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+21, (w5 >> 58) | (w6 << 6) & 0x3ffff, nb,parm);\
- OUT(op,i*32+22, (w6 >> 12) & 0x3ffff, nb,parm);\
- OUT(op,i*32+23, (w6 >> 30) & 0x3ffff, nb,parm); w7 = *(uint64_t *)(ip+(i*9+7)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+24, (w6 >> 48) | (w7 << 16) & 0x3ffff, nb,parm);\
- OUT(op,i*32+25, (w7 >> 2) & 0x3ffff, nb,parm);\
- OUT(op,i*32+26, (w7 >> 20) & 0x3ffff, nb,parm);\
- OUT(op,i*32+27, (w7 >> 38) & 0x3ffff, nb,parm); w8 = *(uint64_t *)(ip+(i*9+8)*8/sizeof(ip[0]));\
-\
- OUT(op,i*32+28, (w7 >> 56) | (w8 << 8) & 0x3ffff, nb,parm);\
- OUT(op,i*32+29, (w8 >> 10) & 0x3ffff, nb,parm);\
- OUT(op,i*32+30, (w8 >> 28) & 0x3ffff, nb,parm);\
- OUT(op,i*32+31, (w8 >> 46) , nb,parm);;\
-}
-
-#define BITUNPACK64_18(ip, op, nb,parm) { \
- BITUNBLK64_18(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 18*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_19(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7,w8,w9; w0 = *(uint64_t *)(ip+(i*19+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 19) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 38) & 0x7ffff, nb,parm); w1 = *(uint64_t *)(ip+(i*19+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 3, (w0 >> 57) | (w1 << 7) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 4, (w1 >> 12) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 5, (w1 >> 31) & 0x7ffff, nb,parm); w2 = *(uint64_t *)(ip+(i*19+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 6, (w1 >> 50) | (w2 << 14) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 7, (w2 >> 5) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 8, (w2 >> 24) & 0x7ffff, nb,parm);\
- OUT(op,i*64+ 9, (w2 >> 43) & 0x7ffff, nb,parm); w3 = *(uint64_t *)(ip+(i*19+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+10, (w2 >> 62) | (w3 << 2) & 0x7ffff, nb,parm);\
- OUT(op,i*64+11, (w3 >> 17) & 0x7ffff, nb,parm);\
- OUT(op,i*64+12, (w3 >> 36) & 0x7ffff, nb,parm); w4 = *(uint64_t *)(ip+(i*19+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+13, (w3 >> 55) | (w4 << 9) & 0x7ffff, nb,parm);\
- OUT(op,i*64+14, (w4 >> 10) & 0x7ffff, nb,parm);\
- OUT(op,i*64+15, (w4 >> 29) & 0x7ffff, nb,parm); w5 = *(uint64_t *)(ip+(i*19+5)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+16, (w4 >> 48) | (w5 << 16) & 0x7ffff, nb,parm);\
- OUT(op,i*64+17, (w5 >> 3) & 0x7ffff, nb,parm);\
- OUT(op,i*64+18, (w5 >> 22) & 0x7ffff, nb,parm);\
- OUT(op,i*64+19, (w5 >> 41) & 0x7ffff, nb,parm); w6 = *(uint64_t *)(ip+(i*19+6)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+20, (w5 >> 60) | (w6 << 4) & 0x7ffff, nb,parm);\
- OUT(op,i*64+21, (w6 >> 15) & 0x7ffff, nb,parm);\
- OUT(op,i*64+22, (w6 >> 34) & 0x7ffff, nb,parm); w7 = *(uint64_t *)(ip+(i*19+7)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+23, (w6 >> 53) | (w7 << 11) & 0x7ffff, nb,parm);\
- OUT(op,i*64+24, (w7 >> 8) & 0x7ffff, nb,parm);\
- OUT(op,i*64+25, (w7 >> 27) & 0x7ffff, nb,parm); w8 = *(uint64_t *)(ip+(i*19+8)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+26, (w7 >> 46) | (w8 << 18) & 0x7ffff, nb,parm);\
- OUT(op,i*64+27, (w8 >> 1) & 0x7ffff, nb,parm);\
- OUT(op,i*64+28, (w8 >> 20) & 0x7ffff, nb,parm);\
- OUT(op,i*64+29, (w8 >> 39) & 0x7ffff, nb,parm); w9 = *(uint32_t *)(ip+(i*19+9)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+30, (w8 >> 58) | (w9 << 6) & 0x7ffff, nb,parm);\
- OUT(op,i*64+31, (w9 >> 13) & 0x7ffff, nb,parm);;\
-}
-
-#define BITUNPACK64_19(ip, op, nb,parm) { \
- BITUNBLK64_19(ip, 0, op, nb,parm); OPI(op, nb,parm); ip += 19*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_20(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7,w8,w9; w0 = *(uint64_t *)(ip+(i*5+0)*8/sizeof(ip[0]));\
- OUT(op,i*16+ 0, (w0 ) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 1, (w0 >> 20) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 2, (w0 >> 40) & 0xfffff, nb,parm); w1 = *(uint64_t *)(ip+(i*5+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+ 3, (w0 >> 60) | (w1 << 4) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 4, (w1 >> 16) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 5, (w1 >> 36) & 0xfffff, nb,parm); w2 = *(uint64_t *)(ip+(i*5+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+ 6, (w1 >> 56) | (w2 << 8) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 7, (w2 >> 12) & 0xfffff, nb,parm);\
- OUT(op,i*16+ 8, (w2 >> 32) & 0xfffff, nb,parm); w3 = *(uint64_t *)(ip+(i*5+3)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+ 9, (w2 >> 52) | (w3 << 12) & 0xfffff, nb,parm);\
- OUT(op,i*16+10, (w3 >> 8) & 0xfffff, nb,parm);\
- OUT(op,i*16+11, (w3 >> 28) & 0xfffff, nb,parm); w4 = *(uint64_t *)(ip+(i*5+4)*8/sizeof(ip[0]));\
-\
- OUT(op,i*16+12, (w3 >> 48) | (w4 << 16) & 0xfffff, nb,parm);\
- OUT(op,i*16+13, (w4 >> 4) & 0xfffff, nb,parm);\
- OUT(op,i*16+14, (w4 >> 24) & 0xfffff, nb,parm);\
- OUT(op,i*16+15, (w4 >> 44) , nb,parm);;\
-}
-
-#define BITUNPACK64_20(ip, op, nb,parm) { \
- BITUNBLK64_20(ip, 0, op, nb,parm);\
- BITUNBLK64_20(ip, 1, op, nb,parm); OPI(op, nb,parm); ip += 20*4/sizeof(ip[0]);\
-}
-
-#define BITUNBLK64_21(ip, i, op, nb,parm) { uint64_t w0,w1,w2,w3,w4,w5,w6,w7,w8,w9,w10; w0 = *(uint64_t *)(ip+(i*21+0)*8/sizeof(ip[0]));\
- OUT(op,i*64+ 0, (w0 ) & 0x1fffff, nb,parm);\
- OUT(op,i*64+ 1, (w0 >> 21) & 0x1fffff, nb,parm);\
- OUT(op,i*64+ 2, (w0 >> 42) & 0x1fffff, nb,parm); w1 = *(uint64_t *)(ip+(i*21+1)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 3, (w0 >> 63) | (w1 << 1) & 0x1fffff, nb,parm);\
- OUT(op,i*64+ 4, (w1 >> 20) & 0x1fffff, nb,parm);\
- OUT(op,i*64+ 5, (w1 >> 41) & 0x1fffff, nb,parm); w2 = *(uint64_t *)(ip+(i*21+2)*8/sizeof(ip[0]));\
-\
- OUT(op,i*64+ 6, (w1 >> 62) | (w2 << 2) & 0x1fffff, nb,parm);\
... 16491 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org