You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2022/07/13 16:28:23 UTC
[arrow] branch master updated: ARROW-16324: [Go] Implement Dictionary Unification (#13529)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 861f23717f ARROW-16324: [Go] Implement Dictionary Unification (#13529)
861f23717f is described below
commit 861f23717f5cffb584bb66ffa66d5a1afb8ab8d7
Author: Matt Topol <mt...@factset.com>
AuthorDate: Wed Jul 13 12:28:16 2022 -0400
ARROW-16324: [Go] Implement Dictionary Unification (#13529)
In addition to adding a `DictionaryUnifier` object, and functions for unifying Chunked arrays and Tables, this also adds the necessary SIMD implemented functions to perform the transposition of the integers (along with a benchmark). Calling `TransposeInts` on an array of 1000 8-bit integers across a 50 element mapping took only 30 ns on my laptop as opposed to around 7000 ns for non-SIMD solution. Pure-Go implementations are also provided for the processors that don't have SSE4 or AVX [...]
Authored-by: Matthew Topol <mt...@factset.com>
Signed-off-by: Matthew Topol <mt...@factset.com>
---
go/arrow/array.go | 3 +
go/arrow/array/binarybuilder.go | 4 +
go/arrow/array/bufferbuilder.go | 2 +-
go/arrow/array/data.go | 7 +-
go/arrow/array/dictionary.go | 427 ++-
go/arrow/array/dictionary_test.go | 457 +++
go/arrow/array/extension.go | 6 +-
go/arrow/array/util.go | 49 +
go/arrow/internal/testing/types/extension_types.go | 40 +
go/internal/hashing/xxh3_memo_table.go | 7 +-
go/internal/utils/Makefile | 17 +-
go/internal/utils/_lib/transpose_ints.c | 57 +
go/internal/utils/_lib/transpose_ints_avx2_amd64.s | 3334 ++++++++++++++++++++
go/internal/utils/_lib/transpose_ints_sse4_amd64.s | 3334 ++++++++++++++++++++
go/internal/utils/transpose_ints.go | 407 +++
go/internal/utils/transpose_ints.go.tmpl | 34 +
go/internal/utils/transpose_ints.tmpldata | 34 +
go/internal/utils/transpose_ints_amd64.go | 326 ++
go/internal/utils/transpose_ints_amd64.go.tmpl | 75 +
go/internal/utils/transpose_ints_arm64.go | 97 +
go/internal/utils/transpose_ints_avx2_amd64.go | 474 +++
go/internal/utils/transpose_ints_avx2_amd64.s | 3074 ++++++++++++++++++
go/internal/utils/transpose_ints_def.go | 227 ++
go/internal/utils/transpose_ints_noasm.go | 97 +
go/internal/utils/transpose_ints_noasm.go.tmpl | 34 +
go/internal/utils/transpose_ints_s390x.go | 97 +
go/internal/utils/transpose_ints_s390x.go.tmpl | 34 +
go/internal/utils/transpose_ints_simd.go.tmpl | 42 +
go/internal/utils/transpose_ints_sse4_amd64.go | 474 +++
go/internal/utils/transpose_ints_sse4_amd64.s | 3074 ++++++++++++++++++
go/internal/utils/transpose_ints_test.go | 50 +
31 files changed, 16339 insertions(+), 55 deletions(-)
diff --git a/go/arrow/array.go b/go/arrow/array.go
index dfefe88c29..0742ae11f0 100644
--- a/go/arrow/array.go
+++ b/go/arrow/array.go
@@ -77,6 +77,9 @@ type ArrayData interface {
// Reset allows reusing this ArrayData object by replacing the data in this ArrayData
// object without changing the reference count.
Reset(newtype DataType, newlength int, newbuffers []*memory.Buffer, newchildren []ArrayData, newnulls int, newoffset int)
+ // Dictionary returns the ArrayData object for the dictionary if this is a
+ // dictionary array, otherwise it will be nil.
+ Dictionary() ArrayData
}
// Array represents an immutable sequence of values using the Arrow in-memory format.
diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go
index 81c840d138..5905380787 100644
--- a/go/arrow/array/binarybuilder.go
+++ b/go/arrow/array/binarybuilder.go
@@ -179,6 +179,10 @@ func (b *BinaryBuilder) Resize(n int) {
b.builder.resize(n, b.init)
}
+func (b *BinaryBuilder) ResizeData(n int) {
+ b.values.length = 0
+}
+
// NewArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewArray() arrow.Array {
diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go
index 90d0d18a81..5901526179 100644
--- a/go/arrow/array/bufferbuilder.go
+++ b/go/arrow/array/bufferbuilder.go
@@ -73,7 +73,7 @@ func (b *bufferBuilder) resize(elements int) {
b.buffer = memory.NewResizableBuffer(b.mem)
}
- b.buffer.Resize(elements)
+ b.buffer.ResizeNoShrink(elements)
oldCapacity := b.capacity
b.capacity = b.buffer.Cap()
b.bytes = b.buffer.Buf()
diff --git a/go/arrow/array/data.go b/go/arrow/array/data.go
index c333555bae..b7a1993c1a 100644
--- a/go/arrow/array/data.go
+++ b/go/arrow/array/data.go
@@ -165,11 +165,14 @@ func (d *Data) Dictionary() arrow.ArrayData { return d.dictionary }
// SetDictionary allows replacing the dictionary for this particular Data object
func (d *Data) SetDictionary(dict arrow.ArrayData) {
- dict.Retain()
if d.dictionary != nil {
d.dictionary.Release()
+ d.dictionary = nil
+ }
+ if dict.(*Data) != nil {
+ dict.Retain()
+ d.dictionary = dict.(*Data)
}
- d.dictionary = dict.(*Data)
}
// NewSliceData returns a new slice that shares backing data with the input.
diff --git a/go/arrow/array/dictionary.go b/go/arrow/array/dictionary.go
index 05ce4f8f36..7894e873bb 100644
--- a/go/arrow/array/dictionary.go
+++ b/go/arrow/array/dictionary.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"errors"
"fmt"
"math"
@@ -387,7 +388,6 @@ func createMemoTable(mem memory.Allocator, dt arrow.DataType) (ret hashing.MemoT
ret = hashing.NewBinaryMemoTable(0, 0, NewBinaryBuilder(mem, arrow.BinaryTypes.String))
case arrow.NULL:
default:
- debug.Assert(false, "unimplemented dictionary value type")
err = fmt.Errorf("unimplemented dictionary value type, %s", dt)
}
@@ -693,12 +693,31 @@ func (b *dictionaryBuilder) ResetFull() {
func (b *dictionaryBuilder) Cap() int { return b.idxBuilder.Cap() }
// UnmarshalJSON is not yet implemented for dictionary builders and will always error.
-func (b *dictionaryBuilder) UnmarshalJSON([]byte) error {
- return errors.New("unmarshal json to dictionary not yet implemented")
+func (b *dictionaryBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("dictionary builder must upack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
}
func (b *dictionaryBuilder) unmarshal(dec *json.Decoder) error {
- return errors.New("unmarshal json to dictionary not yet implemented")
+ bldr := NewBuilder(b.mem, b.dt.ValueType)
+ defer bldr.Release()
+
+ if err := bldr.unmarshal(dec); err != nil {
+ return err
+ }
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+ return b.AppendArray(arr)
}
func (b *dictionaryBuilder) unmarshalOne(dec *json.Decoder) error {
@@ -732,49 +751,8 @@ func (b *dictionaryBuilder) newWithDictOffset(offset int) (indices, dict *Data,
indices = idxarr.Data().(*Data)
indices.Retain()
- dictBuffers := make([]*memory.Buffer, 2)
-
- dictLength := b.memoTable.Size() - offset
- dictBuffers[1] = memory.NewResizableBuffer(b.mem)
- defer dictBuffers[1].Release()
-
- if bintbl, ok := b.memoTable.(*hashing.BinaryMemoTable); ok {
- switch b.dt.ValueType.ID() {
- case arrow.BINARY, arrow.STRING:
- dictBuffers = append(dictBuffers, memory.NewResizableBuffer(b.mem))
- defer dictBuffers[2].Release()
-
- dictBuffers[1].Resize(arrow.Int32SizeBytes * (dictLength + 1))
- offsets := arrow.Int32Traits.CastFromBytes(dictBuffers[1].Bytes())
- bintbl.CopyOffsetsSubset(offset, offsets)
-
- valuesz := offsets[len(offsets)-1] - offsets[0]
- dictBuffers[2].Resize(int(valuesz))
- bintbl.CopyValuesSubset(offset, dictBuffers[2].Bytes())
- default: // fixed size
- bw := int(bitutil.BytesForBits(int64(b.dt.ValueType.(arrow.FixedWidthDataType).BitWidth())))
- dictBuffers[1].Resize(dictLength * bw)
- bintbl.CopyFixedWidthValues(offset, bw, dictBuffers[1].Bytes())
- }
- } else {
- dictBuffers[1].Resize(b.memoTable.TypeTraits().BytesRequired(dictLength))
- b.memoTable.WriteOutSubset(offset, dictBuffers[1].Bytes())
- }
-
- var nullcount int
- if idx, ok := b.memoTable.GetNull(); ok && idx >= offset {
- dictBuffers[0] = memory.NewResizableBuffer(b.mem)
- defer dictBuffers[0].Release()
-
- nullcount = 1
-
- dictBuffers[0].Resize(int(bitutil.BytesForBits(int64(dictLength))))
- memory.Set(dictBuffers[0].Bytes(), 0xFF)
- bitutil.ClearBit(dictBuffers[0].Bytes(), idx)
- }
-
b.deltaOffset = b.memoTable.Size()
- dict = NewData(b.dt.ValueType, dictLength, dictBuffers, nil, nullcount, 0)
+ dict, err = getDictArrayData(b.mem, b.dt.ValueType, b.memoTable, offset)
b.reset()
return
}
@@ -1288,6 +1266,363 @@ func (b *DayTimeDictionaryBuilder) InsertDictValues(arr *DayTimeInterval) (err e
return
}
+func IsTrivialTransposition(transposeMap []int32) bool {
+ for i, t := range transposeMap {
+ if t != int32(i) {
+ return false
+ }
+ }
+ return true
+}
+
+func TransposeDictIndices(mem memory.Allocator, data arrow.ArrayData, inType, outType arrow.DataType, dict arrow.ArrayData, transposeMap []int32) (arrow.ArrayData, error) {
+ // inType may be different from data->dtype if data is ExtensionType
+ if inType.ID() != arrow.DICTIONARY || outType.ID() != arrow.DICTIONARY {
+ return nil, errors.New("arrow/array: expected dictionary type")
+ }
+
+ var (
+ inDictType = inType.(*arrow.DictionaryType)
+ outDictType = outType.(*arrow.DictionaryType)
+ inIndexType = inDictType.IndexType
+ outIndexType = outDictType.IndexType.(arrow.FixedWidthDataType)
+ )
+
+ if inIndexType.ID() == outIndexType.ID() && IsTrivialTransposition(transposeMap) {
+ // index type and values will be identical, we can reuse the existing buffers
+ return NewDataWithDictionary(outType, data.Len(), []*memory.Buffer{data.Buffers()[0], data.Buffers()[1]},
+ data.NullN(), data.Offset(), dict.(*Data)), nil
+ }
+
+ // default path: compute the transposed indices as a new buffer
+ outBuf := memory.NewResizableBuffer(mem)
+ outBuf.Resize(data.Len() * int(bitutil.BytesForBits(int64(outIndexType.BitWidth()))))
+ defer outBuf.Release()
+
+ // shift null buffer if original offset is non-zero
+ var nullBitmap *memory.Buffer
+ if data.Offset() != 0 && data.NullN() != 0 {
+ nullBitmap = memory.NewResizableBuffer(mem)
+ nullBitmap.Resize(int(bitutil.BytesForBits(int64(data.Len()))))
+ bitutil.CopyBitmap(data.Buffers()[0].Bytes(), data.Offset(), data.Len(), nullBitmap.Bytes(), 0)
+ defer nullBitmap.Release()
+ } else {
+ nullBitmap = data.Buffers()[0]
+ }
+
+ outData := NewDataWithDictionary(outType, data.Len(),
+ []*memory.Buffer{nullBitmap, outBuf}, data.NullN(), 0, dict.(*Data))
+ err := utils.TransposeIntsBuffers(inIndexType, outIndexType,
+ data.Buffers()[1].Bytes(), outBuf.Bytes(), data.Offset(), outData.offset, data.Len(), transposeMap)
+ return outData, err
+}
+
+// DictionaryUnifier defines the interface used for unifying, and optionally producing
+// transposition maps for, multiple dictionary arrays incrementally.
+type DictionaryUnifier interface {
+ // Unify adds the provided array of dictionary values to be unified.
+ Unify(arrow.Array) error
+ // UnifyAndTranspose adds the provided array of dictionary values,
+ // just like Unify but returns an allocated buffer containing a mapping
+ // to transpose dictionary indices.
+ UnifyAndTranspose(dict arrow.Array) (transposed *memory.Buffer, err error)
+ // GetResult returns the dictionary type (choosing the smallest index type
+ // that can represent all the values) and the new unified dictionary.
+ //
+ // Calling GetResult clears the existing dictionary from the unifier so it
+ // can be reused by calling Unify/UnifyAndTranspose again with new arrays.
+ GetResult() (outType arrow.DataType, outDict arrow.Array, err error)
+ // GetResultWithIndexType is like GetResult, but allows specifying the type
+ // of the dictionary indexes rather than letting the unifier pick. If the
+ // passed in index type isn't large enough to represent all of the dictionary
+ // values, an error will be returned instead. The new unified dictionary
+ // is returned.
+ GetResultWithIndexType(indexType arrow.DataType) (arrow.Array, error)
+ // Release should be called to clean up any allocated scrach memo-table used
+ // for building the unified dictionary.
+ Release()
+}
+
+type unifier struct {
+ mem memory.Allocator
+ valueType arrow.DataType
+ memoTable hashing.MemoTable
+}
+
+// NewDictionaryUnifier constructs and returns a new dictionary unifier for dictionaries
+// of valueType, using the provided allocator for allocating the unified dictionary
+// and the memotable used for building it.
+//
+// This will only work for non-nested types currently. a nested valueType or dictionary type
+// will result in an error.
+func NewDictionaryUnifier(alloc memory.Allocator, valueType arrow.DataType) (DictionaryUnifier, error) {
+ memoTable, err := createMemoTable(alloc, valueType)
+ if err != nil {
+ return nil, err
+ }
+ return &unifier{
+ mem: alloc,
+ valueType: valueType,
+ memoTable: memoTable,
+ }, nil
+}
+
+func (u *unifier) Release() {
+ if bin, ok := u.memoTable.(*hashing.BinaryMemoTable); ok {
+ bin.Release()
+ }
+}
+
+func (u *unifier) Unify(dict arrow.Array) (err error) {
+ if !arrow.TypeEqual(u.valueType, dict.DataType()) {
+ return fmt.Errorf("dictionary type different from unifier: %s, expected: %s", dict.DataType(), u.valueType)
+ }
+
+ valFn := getvalFn(dict)
+ for i := 0; i < dict.Len(); i++ {
+ if dict.IsNull(i) {
+ u.memoTable.GetOrInsertNull()
+ continue
+ }
+
+ if _, _, err = u.memoTable.GetOrInsert(valFn(i)); err != nil {
+ return err
+ }
+ }
+ return
+}
+
+func (u *unifier) UnifyAndTranspose(dict arrow.Array) (transposed *memory.Buffer, err error) {
+ if !arrow.TypeEqual(u.valueType, dict.DataType()) {
+ return nil, fmt.Errorf("dictionary type different from unifier: %s, expected: %s", dict.DataType(), u.valueType)
+ }
+
+ transposed = memory.NewResizableBuffer(u.mem)
+ transposed.Resize(arrow.Int32Traits.BytesRequired(dict.Len()))
+
+ newIdxes := arrow.Int32Traits.CastFromBytes(transposed.Bytes())
+ valFn := getvalFn(dict)
+ for i := 0; i < dict.Len(); i++ {
+ if dict.IsNull(i) {
+ idx, _ := u.memoTable.GetOrInsertNull()
+ newIdxes[i] = int32(idx)
+ continue
+ }
+
+ idx, _, err := u.memoTable.GetOrInsert(valFn(i))
+ if err != nil {
+ transposed.Release()
+ return nil, err
+ }
+ newIdxes[i] = int32(idx)
+ }
+ return
+}
+
+func (u *unifier) GetResult() (outType arrow.DataType, outDict arrow.Array, err error) {
+ dictLen := u.memoTable.Size()
+ var indexType arrow.DataType
+ switch {
+ case dictLen <= math.MaxInt8:
+ indexType = arrow.PrimitiveTypes.Int8
+ case dictLen <= math.MaxInt16:
+ indexType = arrow.PrimitiveTypes.Int16
+ case dictLen <= math.MaxInt32:
+ indexType = arrow.PrimitiveTypes.Int32
+ default:
+ indexType = arrow.PrimitiveTypes.Int64
+ }
+ outType = &arrow.DictionaryType{IndexType: indexType, ValueType: u.valueType}
+
+ dictData, err := getDictArrayData(u.mem, u.valueType, u.memoTable, 0)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ u.memoTable.Reset()
+
+ defer dictData.Release()
+ outDict = MakeFromData(dictData)
+ return
+}
+
+func (u *unifier) GetResultWithIndexType(indexType arrow.DataType) (arrow.Array, error) {
+ dictLen := u.memoTable.Size()
+ var toobig bool
+ switch indexType.ID() {
+ case arrow.UINT8:
+ toobig = dictLen > math.MaxUint8
+ case arrow.INT8:
+ toobig = dictLen > math.MaxInt8
+ case arrow.UINT16:
+ toobig = dictLen > math.MaxUint16
+ case arrow.INT16:
+ toobig = dictLen > math.MaxInt16
+ case arrow.UINT32:
+ toobig = dictLen > math.MaxUint32
+ case arrow.INT32:
+ toobig = dictLen > math.MaxInt32
+ case arrow.UINT64:
+ toobig = uint64(dictLen) > uint64(math.MaxUint64)
+ case arrow.INT64:
+ default:
+ return nil, fmt.Errorf("arrow/array: invalid dictionary index type: %s, must be integral", indexType)
+ }
+ if toobig {
+ return nil, errors.New("arrow/array: cannot combine dictionaries. unified dictionary requires a larger index type")
+ }
+
+ dictData, err := getDictArrayData(u.mem, u.valueType, u.memoTable, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ u.memoTable.Reset()
+
+ defer dictData.Release()
+ return MakeFromData(dictData), nil
+}
+
+func unifyRecursive(mem memory.Allocator, typ arrow.DataType, chunks []*Data) (changed bool, err error) {
+ debug.Assert(len(chunks) != 0, "must provide non-zero length chunk slice")
+ var extType arrow.DataType
+
+ if typ.ID() == arrow.EXTENSION {
+ extType = typ
+ typ = typ.(arrow.ExtensionType).StorageType()
+ }
+
+ if nestedTyp, ok := typ.(arrow.NestedType); ok {
+ children := make([]*Data, len(chunks))
+ for i, f := range nestedTyp.Fields() {
+ for j, c := range chunks {
+ children[j] = c.childData[i].(*Data)
+ }
+
+ childChanged, err := unifyRecursive(mem, f.Type, children)
+ if err != nil {
+ return false, err
+ }
+ if childChanged {
+ // only when unification actually occurs
+ for j := range chunks {
+ chunks[j].childData[i] = children[j]
+ }
+ changed = true
+ }
+ }
+ }
+
+ if typ.ID() == arrow.DICTIONARY {
+ dictType := typ.(*arrow.DictionaryType)
+ var (
+ uni DictionaryUnifier
+ newDict arrow.Array
+ )
+ // unify any nested dictionaries first, but the unifier doesn't support
+ // nested dictionaries yet so this would fail.
+ uni, err = NewDictionaryUnifier(mem, dictType.ValueType)
+ if err != nil {
+ return changed, err
+ }
+ defer uni.Release()
+ transposeMaps := make([]*memory.Buffer, len(chunks))
+ for i, c := range chunks {
+ debug.Assert(c.dictionary != nil, "missing dictionary data for dictionary array")
+ arr := MakeFromData(c.dictionary)
+ defer arr.Release()
+ if transposeMaps[i], err = uni.UnifyAndTranspose(arr); err != nil {
+ return
+ }
+ defer transposeMaps[i].Release()
+ }
+
+ if newDict, err = uni.GetResultWithIndexType(dictType.IndexType); err != nil {
+ return
+ }
+ defer newDict.Release()
+
+ for j := range chunks {
+ chnk, err := TransposeDictIndices(mem, chunks[j], typ, typ, newDict.Data(), arrow.Int32Traits.CastFromBytes(transposeMaps[j].Bytes()))
+ if err != nil {
+ return changed, err
+ }
+ chunks[j].Release()
+ chunks[j] = chnk.(*Data)
+ if extType != nil {
+ chunks[j].dtype = extType
+ }
+ }
+ changed = true
+ }
+
+ return
+}
+
+// UnifyChunkedDicts takes a chunked array of dictionary type and will unify
+// the dictionary across all of the chunks with the returned chunked array
+// having all chunks share the same dictionary.
+//
+// The return from this *must* have Release called on it unless an error is returned
+// in which case the *arrow.Chunked will be nil.
+//
+// If there is 1 or fewer chunks, then nothing is modified and this function will just
+// call Retain on the passed in Chunked array (so Release can safely be called on it).
+// The same is true if the type of the array is not a dictionary or if no changes are
+// needed for all of the chunks to be using the same dictionary.
+func UnifyChunkedDicts(alloc memory.Allocator, chnkd *arrow.Chunked) (*arrow.Chunked, error) {
+ if len(chnkd.Chunks()) <= 1 {
+ chnkd.Retain()
+ return chnkd, nil
+ }
+
+ chunksData := make([]*Data, len(chnkd.Chunks()))
+ for i, c := range chnkd.Chunks() {
+ c.Data().Retain()
+ chunksData[i] = c.Data().(*Data)
+ }
+ changed, err := unifyRecursive(alloc, chnkd.DataType(), chunksData)
+ if err != nil || !changed {
+ for _, c := range chunksData {
+ c.Release()
+ }
+ if err == nil {
+ chnkd.Retain()
+ } else {
+ chnkd = nil
+ }
+ return chnkd, err
+ }
+
+ chunks := make([]arrow.Array, len(chunksData))
+ for i, c := range chunksData {
+ chunks[i] = MakeFromData(c)
+ defer chunks[i].Release()
+ c.Release()
+ }
+
+ return arrow.NewChunked(chnkd.DataType(), chunks), nil
+}
+
+// UnifyTableDicts performs UnifyChunkedDicts on each column of the table so that
+// any dictionary column will have the dictionaries of its chunks unified.
+//
+// The returned Table should always be Release'd unless a non-nil error was returned,
+// in which case the table returned will be nil.
+func UnifyTableDicts(alloc memory.Allocator, table arrow.Table) (arrow.Table, error) {
+ cols := make([]arrow.Column, table.NumCols())
+ for i := 0; i < int(table.NumCols()); i++ {
+ chnkd, err := UnifyChunkedDicts(alloc, table.Column(i).Data())
+ if err != nil {
+ return nil, err
+ }
+ defer chnkd.Release()
+ cols[i] = *arrow.NewColumn(table.Schema().Field(i), chnkd)
+ defer cols[i].Release()
+ }
+ return NewTable(table.Schema(), cols, table.NumRows()), nil
+}
+
var (
_ arrow.Array = (*Dictionary)(nil)
_ Builder = (*dictionaryBuilder)(nil)
diff --git a/go/arrow/array/dictionary_test.go b/go/arrow/array/dictionary_test.go
index 0fb5c2635a..376d6cbb7c 100644
--- a/go/arrow/array/dictionary_test.go
+++ b/go/arrow/array/dictionary_test.go
@@ -27,8 +27,10 @@ import (
"github.com/apache/arrow/go/v9/arrow/array"
"github.com/apache/arrow/go/v9/arrow/bitutil"
"github.com/apache/arrow/go/v9/arrow/decimal128"
+ "github.com/apache/arrow/go/v9/arrow/internal/testing/types"
"github.com/apache/arrow/go/v9/arrow/memory"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@@ -1181,3 +1183,458 @@ func TestDictionaryGetValueIndex(t *testing.T) {
})
}
}
+
+func checkTransposeMap(t *testing.T, b *memory.Buffer, exp []int32) bool {
+ got := arrow.Int32Traits.CastFromBytes(b.Bytes())
+ return assert.Equal(t, exp, got)
+}
+
+func TestDictionaryUnifierNumeric(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := arrow.PrimitiveTypes.Int64
+
+ d1, _, err := array.FromJSON(mem, dictType, strings.NewReader(`[3, 4, 7]`))
+ require.NoError(t, err)
+ d2, _, err := array.FromJSON(mem, dictType, strings.NewReader(`[1, 7, 4, 8]`))
+ require.NoError(t, err)
+ d3, _, err := array.FromJSON(mem, dictType, strings.NewReader(`[1, -200]`))
+ require.NoError(t, err)
+
+ expected := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: dictType}
+ expectedDict, _, err := array.FromJSON(mem, dictType, strings.NewReader(`[3, 4, 7, 1, 8, -200]`))
+ require.NoError(t, err)
+ defer func() {
+ d1.Release()
+ d2.Release()
+ d3.Release()
+ expectedDict.Release()
+ }()
+
+ unifier, err := array.NewDictionaryUnifier(mem, dictType)
+ assert.NoError(t, err)
+ defer unifier.Release()
+
+ assert.NoError(t, unifier.Unify(d1))
+ assert.NoError(t, unifier.Unify(d2))
+ assert.NoError(t, unifier.Unify(d3))
+
+ invalid, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int32, strings.NewReader(`[1, -200]`))
+ defer invalid.Release()
+ assert.EqualError(t, unifier.Unify(invalid), "dictionary type different from unifier: int32, expected: int64")
+
+ outType, outDict, err := unifier.GetResult()
+ assert.NoError(t, err)
+ defer outDict.Release()
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ b1, err := unifier.UnifyAndTranspose(d1)
+ assert.NoError(t, err)
+ b2, err := unifier.UnifyAndTranspose(d2)
+ assert.NoError(t, err)
+ b3, err := unifier.UnifyAndTranspose(d3)
+ assert.NoError(t, err)
+
+ outType, outDict, err = unifier.GetResult()
+ assert.NoError(t, err)
+ defer func() {
+ outDict.Release()
+ b1.Release()
+ b2.Release()
+ b3.Release()
+ }()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ checkTransposeMap(t, b1, []int32{0, 1, 2})
+ checkTransposeMap(t, b2, []int32{3, 2, 1, 4})
+ checkTransposeMap(t, b3, []int32{3, 5})
+}
+
+func TestDictionaryUnifierString(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := arrow.BinaryTypes.String
+ d1, _, err := array.FromJSON(mem, dictType, strings.NewReader(`["foo", "bar"]`))
+ require.NoError(t, err)
+ defer d1.Release()
+
+ d2, _, err := array.FromJSON(mem, dictType, strings.NewReader(`["quux", "foo"]`))
+ require.NoError(t, err)
+ defer d2.Release()
+
+ expected := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: dictType}
+ expectedDict, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`["foo", "bar", "quux"]`))
+ defer expectedDict.Release()
+
+ unifier, err := array.NewDictionaryUnifier(mem, dictType)
+ assert.NoError(t, err)
+ defer unifier.Release()
+
+ assert.NoError(t, unifier.Unify(d1))
+ assert.NoError(t, unifier.Unify(d2))
+ outType, outDict, err := unifier.GetResult()
+ assert.NoError(t, err)
+ defer outDict.Release()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ b1, err := unifier.UnifyAndTranspose(d1)
+ assert.NoError(t, err)
+ b2, err := unifier.UnifyAndTranspose(d2)
+ assert.NoError(t, err)
+
+ outType, outDict, err = unifier.GetResult()
+ assert.NoError(t, err)
+ defer func() {
+ outDict.Release()
+ b1.Release()
+ b2.Release()
+ }()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ checkTransposeMap(t, b1, []int32{0, 1})
+ checkTransposeMap(t, b2, []int32{2, 0})
+}
+
+func TestDictionaryUnifierFixedSizeBinary(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.FixedSizeBinaryType{ByteWidth: 3}
+ data := memory.NewBufferBytes([]byte(`foobarbazqux`))
+
+ fsbData := array.NewData(dictType, 2, []*memory.Buffer{nil, memory.SliceBuffer(data, 0, 6)}, nil, 0, 0)
+ defer fsbData.Release()
+ d1 := array.NewFixedSizeBinaryData(fsbData)
+ fsbData = array.NewData(dictType, 3, []*memory.Buffer{nil, memory.SliceBuffer(data, 3, 9)}, nil, 0, 0)
+ defer fsbData.Release()
+ d2 := array.NewFixedSizeBinaryData(fsbData)
+
+ fsbData = array.NewData(dictType, 4, []*memory.Buffer{nil, data}, nil, 0, 0)
+ defer fsbData.Release()
+ expectedDict := array.NewFixedSizeBinaryData(fsbData)
+ expected := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: dictType}
+
+ unifier, err := array.NewDictionaryUnifier(mem, dictType)
+ assert.NoError(t, err)
+
+ defer func() {
+ d1.Release()
+ d2.Release()
+ expectedDict.Release()
+ unifier.Release()
+ }()
+
+ assert.NoError(t, unifier.Unify(d1))
+ assert.NoError(t, unifier.Unify(d2))
+ outType, outDict, err := unifier.GetResult()
+ assert.NoError(t, err)
+ defer outDict.Release()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ b1, err := unifier.UnifyAndTranspose(d1)
+ assert.NoError(t, err)
+ b2, err := unifier.UnifyAndTranspose(d2)
+ assert.NoError(t, err)
+
+ outType, outDict, err = unifier.GetResult()
+ assert.NoError(t, err)
+ defer func() {
+ outDict.Release()
+ b1.Release()
+ b2.Release()
+ }()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+
+ checkTransposeMap(t, b1, []int32{0, 1})
+ checkTransposeMap(t, b2, []int32{1, 2, 3})
+}
+
+func TestDictionaryUnifierLarge(t *testing.T) {
+ // unifying larger dictionaries should choose the right index type
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewInt32Builder(mem)
+ defer bldr.Release()
+ bldr.Reserve(120)
+ for i := int32(0); i < 120; i++ {
+ bldr.UnsafeAppend(i)
+ }
+
+ d1 := bldr.NewInt32Array()
+ defer d1.Release()
+ assert.EqualValues(t, 120, d1.Len())
+
+ bldr.Reserve(30)
+ for i := int32(110); i < 140; i++ {
+ bldr.UnsafeAppend(i)
+ }
+
+ d2 := bldr.NewInt32Array()
+ defer d2.Release()
+ assert.EqualValues(t, 30, d2.Len())
+
+ bldr.Reserve(140)
+ for i := int32(0); i < 140; i++ {
+ bldr.UnsafeAppend(i)
+ }
+
+ expectedDict := bldr.NewInt32Array()
+ defer expectedDict.Release()
+ assert.EqualValues(t, 140, expectedDict.Len())
+
+ // int8 would be too narrow to hold all the values
+ expected := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int16, ValueType: arrow.PrimitiveTypes.Int32}
+
+ unifier, err := array.NewDictionaryUnifier(mem, arrow.PrimitiveTypes.Int32)
+ assert.NoError(t, err)
+ defer unifier.Release()
+
+ assert.NoError(t, unifier.Unify(d1))
+ assert.NoError(t, unifier.Unify(d2))
+ outType, outDict, err := unifier.GetResult()
+ assert.NoError(t, err)
+ defer outDict.Release()
+
+ assert.Truef(t, arrow.TypeEqual(expected, outType), "got: %s, expected: %s", outType, expected)
+ assert.Truef(t, array.Equal(expectedDict, outDict), "got: %s, expected: %s", outDict, expectedDict)
+}
+
+func checkDictionaryArray(t *testing.T, arr, expectedVals, expectedIndices arrow.Array) bool {
+ require.IsType(t, (*array.Dictionary)(nil), arr)
+ dictArr := arr.(*array.Dictionary)
+ ret := true
+ ret = ret && assert.Truef(t, array.Equal(expectedVals, dictArr.Dictionary()), "got: %s, expected: %s", dictArr.Dictionary(), expectedVals)
+ return ret && assert.Truef(t, array.Equal(expectedIndices, dictArr.Indices()), "got: %s, expected: %s", dictArr.Indices(), expectedIndices)
+}
+
+func TestDictionaryUnifierSimpleChunkedArray(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String}
+ chunk1, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`["ab", "cd", null, "cd"]`))
+ chunk2, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`["ef", "cd", "ef"]`))
+ chunk3, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`["ef", "ab", null, "ab"]`))
+ chunk4, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`[]`))
+ chunked := arrow.NewChunked(dictType, []arrow.Array{chunk1, chunk2, chunk3, chunk4})
+ defer func() {
+ chunk1.Release()
+ chunk2.Release()
+ chunk3.Release()
+ chunk4.Release()
+ chunked.Release()
+ }()
+
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ defer unified.Release()
+
+ assert.Len(t, unified.Chunks(), 4)
+ expectedDict, _, _ := array.FromJSON(mem, dictType.ValueType, strings.NewReader(`["ab", "cd", "ef"]`))
+ defer expectedDict.Release()
+
+ c1Indices, _, _ := array.FromJSON(mem, dictType.IndexType, strings.NewReader(`[0, 1, null, 1]`))
+ defer c1Indices.Release()
+ c2Indices, _, _ := array.FromJSON(mem, dictType.IndexType, strings.NewReader(`[2, 1, 2]`))
+ defer c2Indices.Release()
+ c3Indices, _, _ := array.FromJSON(mem, dictType.IndexType, strings.NewReader(`[2, 0, null, 0]`))
+ defer c3Indices.Release()
+ c4Indices, _, _ := array.FromJSON(mem, dictType.IndexType, strings.NewReader(`[]`))
+ defer c4Indices.Release()
+ checkDictionaryArray(t, unified.Chunk(0), expectedDict, c1Indices)
+ checkDictionaryArray(t, unified.Chunk(1), expectedDict, c2Indices)
+ checkDictionaryArray(t, unified.Chunk(2), expectedDict, c3Indices)
+ checkDictionaryArray(t, unified.Chunk(3), expectedDict, c4Indices)
+}
+
+func TestDictionaryUnifierChunkedArrayZeroChunks(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String}
+ chunked := arrow.NewChunked(dictType, []arrow.Array{})
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ assert.True(t, array.ChunkedEqual(unified, chunked))
+}
+
+func TestDictionaryUnifierChunkedArrayOneChunk(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String}
+ chunk1, _, _ := array.FromJSON(mem, dictType, strings.NewReader(`["ab", "cd", null, "cd"]`))
+ defer chunk1.Release()
+
+ chunked := arrow.NewChunked(dictType, []arrow.Array{chunk1})
+ defer chunked.Release()
+
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ defer unified.Release()
+
+ assert.True(t, array.ChunkedEqual(unified, chunked))
+ assert.Same(t, unified, chunked)
+}
+
+func TestDictionaryUnifierChunkedArrayNoDict(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ typ := arrow.PrimitiveTypes.Int8
+ chunk1, _, _ := array.FromJSON(mem, typ, strings.NewReader(`[1, 1, 2, 3]`))
+ defer chunk1.Release()
+
+ chunk2, _, _ := array.FromJSON(mem, typ, strings.NewReader(`[5, 8, 13]`))
+ defer chunk2.Release()
+
+ chunked := arrow.NewChunked(typ, []arrow.Array{chunk1, chunk2})
+ defer chunked.Release()
+
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ defer unified.Release()
+
+ assert.True(t, array.ChunkedEqual(unified, chunked))
+ assert.Same(t, unified, chunked)
+}
+
+func TestDictionaryUnifierChunkedArrayNested(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ typ := arrow.ListOf(&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int16, ValueType: arrow.BinaryTypes.String})
+ chunk1, _, err := array.FromJSON(mem, typ, strings.NewReader(`[["ab", "cd"], ["cd"]]`))
+ assert.NoError(t, err)
+ // defer chunk1.Release()
+ chunk2, _, err := array.FromJSON(mem, typ, strings.NewReader(`[[], ["ef", "cd", "ef"]]`))
+ assert.NoError(t, err)
+ // defer chunk2.Release()
+ chunked := arrow.NewChunked(typ, []arrow.Array{chunk1, chunk2})
+ // defer chunked.Release()
+
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ // defer unified.Release()
+ assert.Len(t, unified.Chunks(), 2)
+
+ expectedDict, _, _ := array.FromJSON(mem, arrow.BinaryTypes.String, strings.NewReader(`["ab", "cd", "ef"]`))
+ // defer expectedDict.Release()
+
+ unified1 := unified.Chunk(0).(*array.List)
+ assert.Equal(t, []int32{0, 2, 3}, unified1.Offsets())
+ expectedIndices1, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int16, strings.NewReader(`[0, 1, 1]`))
+ // defer expectedIndices1.Release()
+ checkDictionaryArray(t, unified1.ListValues(), expectedDict, expectedIndices1)
+
+ unified2 := unified.Chunk(1).(*array.List)
+ assert.Equal(t, []int32{0, 0, 3}, unified2.Offsets())
+ expectedIndices2, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int16, strings.NewReader(`[2, 1, 2]`))
+ // defer expectedIndices2.Release()
+ checkDictionaryArray(t, unified2.ListValues(), expectedDict, expectedIndices2)
+ defer func() {
+ expectedIndices1.Release()
+ expectedIndices2.Release()
+ expectedDict.Release()
+ unified.Release()
+ chunked.Release()
+ chunk2.Release()
+ chunk1.Release()
+ }()
+}
+
+func TestDictionaryUnifierChunkedArrayExtension(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dt := types.NewDictExtensionType()
+ chunk1, _, err := array.FromJSON(mem, dt, strings.NewReader(`["ab", null, "cd", "ab"]`))
+ assert.NoError(t, err)
+ defer chunk1.Release()
+
+ chunk2, _, err := array.FromJSON(mem, dt, strings.NewReader(`["ef", "ab", "ab"]`))
+ assert.NoError(t, err)
+ defer chunk2.Release()
+
+ chunked := arrow.NewChunked(dt, []arrow.Array{chunk1, chunk2})
+ defer chunked.Release()
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.NoError(t, err)
+ defer unified.Release()
+ assert.Len(t, unified.Chunks(), 2)
+
+ expectedDict, _, _ := array.FromJSON(mem, arrow.BinaryTypes.String, strings.NewReader(`["ab", "cd", "ef"]`))
+ defer expectedDict.Release()
+
+ unified1 := unified.Chunk(0).(array.ExtensionArray)
+ assert.Truef(t, arrow.TypeEqual(dt, unified1.DataType()), "expected: %s, got: %s", dt, unified1.DataType())
+ indices, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int8, strings.NewReader(`[0, null, 1, 0]`))
+ defer indices.Release()
+ checkDictionaryArray(t, unified1.Storage(), expectedDict, indices)
+
+ unified2 := unified.Chunk(1).(array.ExtensionArray)
+ assert.Truef(t, arrow.TypeEqual(dt, unified2.DataType()), "expected: %s, got: %s", dt, unified1.DataType())
+ indices, _, _ = array.FromJSON(mem, arrow.PrimitiveTypes.Int8, strings.NewReader(`[2, 0, 0]`))
+ defer indices.Release()
+ checkDictionaryArray(t, unified2.Storage(), expectedDict, indices)
+}
+
+func TestDictionaryUnifierChunkedArrayNestedDict(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ innerType := arrow.ListOf(&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint32, ValueType: arrow.BinaryTypes.String})
+ innerDict1, _, err := array.FromJSON(mem, innerType, strings.NewReader(`[["ab", "cd"], [], ["cd", null]]`))
+ assert.NoError(t, err)
+ defer innerDict1.Release()
+ indices1, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int32, strings.NewReader(`[2, 1, 0, 1, 2]`))
+ defer indices1.Release()
+
+ chunk1 := array.NewDictionaryArray(&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int32, ValueType: innerType}, indices1, innerDict1)
+ defer chunk1.Release()
+
+ innerDict2, _, err := array.FromJSON(mem, innerType, strings.NewReader(`[["cd", "ef"], ["cd", null], []]`))
+ assert.NoError(t, err)
+ defer innerDict2.Release()
+ indices2, _, _ := array.FromJSON(mem, arrow.PrimitiveTypes.Int32, strings.NewReader(`[1, 2, 2, 0]`))
+ defer indices2.Release()
+
+ chunk2 := array.NewDictionaryArray(&arrow.DictionaryType{IndexType: indices2.DataType(), ValueType: innerType}, indices2, innerDict2)
+ defer chunk2.Release()
+
+ chunked := arrow.NewChunked(chunk1.DataType(), []arrow.Array{chunk1, chunk2})
+ defer chunked.Release()
+
+ unified, err := array.UnifyChunkedDicts(mem, chunked)
+ assert.Nil(t, unified)
+ assert.EqualError(t, err, "unimplemented dictionary value type, list<item: dictionary<values=utf8, indices=uint32, ordered=false>, nullable>")
+}
+
+func TestDictioanryUnifierTableZeroColumns(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ schema := arrow.NewSchema([]arrow.Field{}, nil)
+ table := array.NewTable(schema, []arrow.Column{}, 42)
+ defer table.Release()
+
+ unified, err := array.UnifyTableDicts(mem, table)
+ assert.NoError(t, err)
+ assert.True(t, schema.Equal(unified.Schema()))
+ assert.EqualValues(t, 42, unified.NumRows())
+ assert.True(t, array.TableEqual(table, unified))
+}
diff --git a/go/arrow/array/extension.go b/go/arrow/array/extension.go
index bdc1312fbc..94babc3829 100644
--- a/go/arrow/array/extension.go
+++ b/go/arrow/array/extension.go
@@ -175,6 +175,7 @@ func (e *ExtensionArrayBase) setData(data *Data) {
// our underlying storage needs to reference the same data buffers (no copying)
// but should have the storage type's datatype, so we create a Data for it.
storageData := NewData(extType.StorageType(), data.length, data.buffers, data.childData, data.nulls, data.offset)
+ storageData.SetDictionary(data.dictionary)
defer storageData.Release()
e.storage = MakeFromData(storageData).(arraymarshal)
}
@@ -239,9 +240,8 @@ func (b *ExtensionBuilder) NewExtensionArray() ExtensionArray {
storage := b.Builder.NewArray()
defer storage.Release()
- data := NewData(b.dt, storage.Len(), storage.Data().Buffers(), storage.Data().Children(), storage.Data().NullN(), 0)
- defer data.Release()
- return NewExtensionData(data)
+ storage.Data().(*Data).dtype = b.dt
+ return NewExtensionData(storage.Data())
}
var (
diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go
index 64c8912899..5510b8df17 100644
--- a/go/arrow/array/util.go
+++ b/go/arrow/array/util.go
@@ -22,7 +22,9 @@ import (
"io"
"github.com/apache/arrow/go/v9/arrow"
+ "github.com/apache/arrow/go/v9/arrow/bitutil"
"github.com/apache/arrow/go/v9/arrow/memory"
+ "github.com/apache/arrow/go/v9/internal/hashing"
"github.com/goccy/go-json"
)
@@ -233,3 +235,50 @@ func RecordToJSON(rec arrow.Record, w io.Writer) error {
}
return nil
}
+
+func getDictArrayData(mem memory.Allocator, valueType arrow.DataType, memoTable hashing.MemoTable, startOffset int) (*Data, error) {
+ dictLen := memoTable.Size() - startOffset
+ buffers := []*memory.Buffer{nil, nil}
+
+ buffers[1] = memory.NewResizableBuffer(mem)
+ defer buffers[1].Release()
+
+ switch tbl := memoTable.(type) {
+ case hashing.NumericMemoTable:
+ nbytes := tbl.TypeTraits().BytesRequired(dictLen)
+ buffers[1].Resize(nbytes)
+ tbl.WriteOutSubset(startOffset, buffers[1].Bytes())
+ case *hashing.BinaryMemoTable:
+ switch valueType.ID() {
+ case arrow.BINARY, arrow.STRING:
+ buffers = append(buffers, memory.NewResizableBuffer(mem))
+ defer buffers[2].Release()
+
+ buffers[1].Resize(arrow.Int32Traits.BytesRequired(dictLen + 1))
+ offsets := arrow.Int32Traits.CastFromBytes(buffers[1].Bytes())
+ tbl.CopyOffsetsSubset(startOffset, offsets)
+
+ valuesz := offsets[len(offsets)-1] - offsets[0]
+ buffers[2].Resize(int(valuesz))
+ tbl.CopyValuesSubset(startOffset, buffers[2].Bytes())
+ default: // fixed size
+ bw := int(bitutil.BytesForBits(int64(valueType.(arrow.FixedWidthDataType).BitWidth())))
+ buffers[1].Resize(dictLen * bw)
+ tbl.CopyFixedWidthValues(startOffset, bw, buffers[1].Bytes())
+ }
+ default:
+ return nil, fmt.Errorf("arrow/array: dictionary unifier unimplemented type: %s", valueType)
+ }
+
+ var nullcount int
+ if idx, ok := memoTable.GetNull(); ok && idx >= startOffset {
+ buffers[0] = memory.NewResizableBuffer(mem)
+ defer buffers[0].Release()
+ nullcount = 1
+ buffers[0].Resize(int(bitutil.BytesForBits(int64(dictLen))))
+ memory.Set(buffers[0].Bytes(), 0xFF)
+ bitutil.ClearBit(buffers[0].Bytes(), idx)
+ }
+
+ return NewData(valueType, dictLen, buffers, nil, nullcount, 0), nil
+}
diff --git a/go/arrow/internal/testing/types/extension_types.go b/go/arrow/internal/testing/types/extension_types.go
index 5f9543977d..c266de169e 100644
--- a/go/arrow/internal/testing/types/extension_types.go
+++ b/go/arrow/internal/testing/types/extension_types.go
@@ -235,13 +235,53 @@ func (ExtStructType) Deserialize(_ arrow.DataType, serialized string) (arrow.Ext
return NewExtStructType(), nil
}
+type DictExtensionArray struct {
+ array.ExtensionArrayBase
+}
+
+type DictExtensionType struct {
+ arrow.ExtensionBase
+}
+
+func NewDictExtensionType() *DictExtensionType {
+ return &DictExtensionType{
+ ExtensionBase: arrow.ExtensionBase{
+ Storage: &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String},
+ },
+ }
+}
+
+func (p *DictExtensionType) ExtensionEquals(other arrow.ExtensionType) bool {
+ return other.ExtensionName() == p.ExtensionName()
+}
+
+func (DictExtensionType) ExtensionName() string { return "dict-extension" }
+
+func (DictExtensionType) Serialize() string { return "dict-extension-serialized" }
+
+func (DictExtensionType) ArrayType() reflect.Type { return reflect.TypeOf(DictExtensionArray{}) }
+
+func (p *DictExtensionType) String() string { return "extension<" + p.ExtensionName() + ">" }
+
+func (p *DictExtensionType) Deserialize(storage arrow.DataType, data string) (arrow.ExtensionType, error) {
+ if data != "dict-extension-serialized" {
+ return nil, fmt.Errorf("type identifier did not match: '%s'", data)
+ }
+ if !arrow.TypeEqual(p.StorageType(), storage) {
+ return nil, fmt.Errorf("invalid storage type for DictExtensionType: %s", storage)
+ }
+ return NewDictExtensionType(), nil
+}
+
var (
_ arrow.ExtensionType = (*UUIDType)(nil)
_ arrow.ExtensionType = (*Parametric1Type)(nil)
_ arrow.ExtensionType = (*Parametric2Type)(nil)
_ arrow.ExtensionType = (*ExtStructType)(nil)
+ _ arrow.ExtensionType = (*DictExtensionType)(nil)
_ array.ExtensionArray = (*UUIDArray)(nil)
_ array.ExtensionArray = (*Parametric1Array)(nil)
_ array.ExtensionArray = (*Parametric2Array)(nil)
_ array.ExtensionArray = (*ExtStructArray)(nil)
+ _ array.ExtensionArray = (*DictExtensionArray)(nil)
)
diff --git a/go/internal/hashing/xxh3_memo_table.go b/go/internal/hashing/xxh3_memo_table.go
index 4c0cc17266..0657be8f00 100644
--- a/go/internal/hashing/xxh3_memo_table.go
+++ b/go/internal/hashing/xxh3_memo_table.go
@@ -170,6 +170,7 @@ type BinaryBuilderIFace interface {
ReserveData(int)
Retain()
Resize(int)
+ ResizeData(int)
Release()
DataLen() int
Value(int) []byte
@@ -216,6 +217,7 @@ func (BinaryMemoTable) TypeTraits() TypeTraits {
func (s *BinaryMemoTable) Reset() {
s.tbl.Reset(32)
s.builder.Resize(0)
+ s.builder.ResizeData(0)
s.builder.Reserve(int(32))
s.builder.ReserveData(int(32) * 4)
s.nullIdx = KeyNotFound
@@ -370,12 +372,13 @@ func (b *BinaryMemoTable) CopyOffsetsSubset(start int, out []int32) {
first := b.findOffset(0)
delta := b.findOffset(start)
- for i := start; i < b.Size(); i++ {
+ sz := b.Size()
+ for i := start; i < sz; i++ {
offset := int32(b.findOffset(i) - delta)
out[i-start] = offset
}
- out[b.Size()-start] = int32(b.builder.DataLen() - (int(delta) - int(first)))
+ out[sz-start] = int32(b.builder.DataLen() - (int(delta) - int(first)))
}
// CopyValues copies the raw binary data bytes out, out should be a []byte
diff --git a/go/internal/utils/Makefile b/go/internal/utils/Makefile
index f56e4b65c3..fded9d1d5b 100644
--- a/go/internal/utils/Makefile
+++ b/go/internal/utils/Makefile
@@ -35,7 +35,7 @@ ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -n
.PHONEY: assembly
INTEL_SOURCES := \
- min_max_avx2_amd64.s min_max_sse4_amd64.s
+ min_max_avx2_amd64.s min_max_sse4_amd64.s transpose_ints_avx2_amd64.s transpose_ints_sse4_amd64.s
#
# ARROW-15336: DO NOT add the assembly target for Arm64 (ARM_SOURCES) until c2goasm added the Arm64 support.
@@ -54,12 +54,27 @@ _lib/min_max_sse4_amd64.s: _lib/min_max.c
_lib/min_max_neon.s: _lib/min_max.c
$(CC) -S $(C_FLAGS_NEON) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+_lib/transpose_ints_avx2_amd64.s: _lib/transpose_ints.c
+ $(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/transpose_ints_sse4_amd64.s: _lib/transpose_ints.c
+ $(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/transpose_ints_neon.s: _lib/transpose_ints.c
+ $(CC) -S $(C_FLAGS_NEON) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
min_max_avx2_amd64.s: _lib/min_max_avx2_amd64.s
$(C2GOASM) -a -f $^ $@
min_max_sse4_amd64.s: _lib/min_max_sse4_amd64.s
$(C2GOASM) -a -f $^ $@
+transpose_ints_avx2_amd64.s: _lib/transpose_ints_avx2_amd64.s
+ $(C2GOASM) -a -f $^ $@
+
+transpose_ints_sse4_amd64.s: _lib/transpose_ints_sse4_amd64.s
+ $(C2GOASM) -a -f $^ $@
+
clean:
rm -f $(INTEL_SOURCES)
rm -f $(addprefix _lib/,$(INTEL_SOURCES))
diff --git a/go/internal/utils/_lib/transpose_ints.c b/go/internal/utils/_lib/transpose_ints.c
new file mode 100644
index 0000000000..e37aed62e0
--- /dev/null
+++ b/go/internal/utils/_lib/transpose_ints.c
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <arch.h>
+#include <stdint.h>
+
+#define CREATE_TRANSPOSE(SRC, DEST) \
+ void FULL_NAME(transpose_ ## SRC ## _ ## DEST)(const SRC ## _t* src, DEST ## _t* dest, int length, const int32_t* transpose_map) { \
+ while (length >= 4) { \
+ dest[0] = (DEST ## _t)(transpose_map[src[0]]); \
+ dest[1] = (DEST ## _t)(transpose_map[src[1]]); \
+ dest[2] = (DEST ## _t)(transpose_map[src[2]]); \
+ dest[3] = (DEST ## _t)(transpose_map[src[3]]); \
+ length -= 4; \
+ src += 4; \
+ dest += 4; \
+ } \
+ while (length > 0) { \
+ *dest++ = (DEST ## _t)(transpose_map[*src++]); \
+ --length; \
+ } \
+ }
+
+#define CREATE_TRANSPOSE_ALL_DEST(DEST) \
+ CREATE_TRANSPOSE(uint8, DEST) \
+ CREATE_TRANSPOSE(int8, DEST) \
+ CREATE_TRANSPOSE(uint16, DEST) \
+ CREATE_TRANSPOSE(int16, DEST) \
+ CREATE_TRANSPOSE(uint32, DEST) \
+ CREATE_TRANSPOSE(int32, DEST) \
+ CREATE_TRANSPOSE(uint64, DEST) \
+ CREATE_TRANSPOSE(int64, DEST)
+
+#define CREATE_TRANSPOSE_ALL() \
+ CREATE_TRANSPOSE_ALL_DEST(uint8) \
+ CREATE_TRANSPOSE_ALL_DEST(int8) \
+ CREATE_TRANSPOSE_ALL_DEST(uint16) \
+ CREATE_TRANSPOSE_ALL_DEST(int16) \
+ CREATE_TRANSPOSE_ALL_DEST(uint32) \
+ CREATE_TRANSPOSE_ALL_DEST(int32) \
+ CREATE_TRANSPOSE_ALL_DEST(uint64) \
+ CREATE_TRANSPOSE_ALL_DEST(int64)
+
+CREATE_TRANSPOSE_ALL()
diff --git a/go/internal/utils/_lib/transpose_ints_avx2_amd64.s b/go/internal/utils/_lib/transpose_ints_avx2_amd64.s
new file mode 100644
index 0000000000..6979475674
--- /dev/null
+++ b/go/internal/utils/_lib/transpose_ints_avx2_amd64.s
@@ -0,0 +1,3334 @@
+ .text
+ .intel_syntax noprefix
+ .file "transpose_ints.c"
+ .globl transpose_uint8_uint8_avx2 # -- Begin function transpose_uint8_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_uint8_avx2,@function
+transpose_uint8_uint8_avx2: # @transpose_uint8_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB0_1
+ .p2align 4, 0x90
+.LBB0_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB0_5
+.LBB0_1:
+ test edx, edx
+ jle .LBB0_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB0_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB0_3
+.LBB0_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end0:
+ .size transpose_uint8_uint8_avx2, .Lfunc_end0-transpose_uint8_uint8_avx2
+ # -- End function
+ .globl transpose_int8_uint8_avx2 # -- Begin function transpose_int8_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_uint8_avx2,@function
+transpose_int8_uint8_avx2: # @transpose_int8_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB1_1
+ .p2align 4, 0x90
+.LBB1_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB1_5
+.LBB1_1:
+ test edx, edx
+ jle .LBB1_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB1_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB1_3
+.LBB1_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end1:
+ .size transpose_int8_uint8_avx2, .Lfunc_end1-transpose_int8_uint8_avx2
+ # -- End function
+ .globl transpose_uint16_uint8_avx2 # -- Begin function transpose_uint16_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_uint8_avx2,@function
+transpose_uint16_uint8_avx2: # @transpose_uint16_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB2_1
+ .p2align 4, 0x90
+.LBB2_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB2_5
+.LBB2_1:
+ test edx, edx
+ jle .LBB2_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB2_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB2_3
+.LBB2_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end2:
+ .size transpose_uint16_uint8_avx2, .Lfunc_end2-transpose_uint16_uint8_avx2
+ # -- End function
+ .globl transpose_int16_uint8_avx2 # -- Begin function transpose_int16_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_uint8_avx2,@function
+transpose_int16_uint8_avx2: # @transpose_int16_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB3_1
+ .p2align 4, 0x90
+.LBB3_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB3_5
+.LBB3_1:
+ test edx, edx
+ jle .LBB3_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB3_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB3_3
+.LBB3_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end3:
+ .size transpose_int16_uint8_avx2, .Lfunc_end3-transpose_int16_uint8_avx2
+ # -- End function
+ .globl transpose_uint32_uint8_avx2 # -- Begin function transpose_uint32_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_uint8_avx2,@function
+transpose_uint32_uint8_avx2: # @transpose_uint32_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB4_1
+ .p2align 4, 0x90
+.LBB4_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB4_5
+.LBB4_1:
+ test edx, edx
+ jle .LBB4_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB4_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB4_3
+.LBB4_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end4:
+ .size transpose_uint32_uint8_avx2, .Lfunc_end4-transpose_uint32_uint8_avx2
+ # -- End function
+ .globl transpose_int32_uint8_avx2 # -- Begin function transpose_int32_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_uint8_avx2,@function
+transpose_int32_uint8_avx2: # @transpose_int32_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB5_1
+ .p2align 4, 0x90
+.LBB5_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB5_5
+.LBB5_1:
+ test edx, edx
+ jle .LBB5_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB5_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB5_3
+.LBB5_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end5:
+ .size transpose_int32_uint8_avx2, .Lfunc_end5-transpose_int32_uint8_avx2
+ # -- End function
+ .globl transpose_uint64_uint8_avx2 # -- Begin function transpose_uint64_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_uint8_avx2,@function
+transpose_uint64_uint8_avx2: # @transpose_uint64_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB6_1
+ .p2align 4, 0x90
+.LBB6_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB6_5
+.LBB6_1:
+ test edx, edx
+ jle .LBB6_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB6_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB6_3
+.LBB6_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end6:
+ .size transpose_uint64_uint8_avx2, .Lfunc_end6-transpose_uint64_uint8_avx2
+ # -- End function
+ .globl transpose_int64_uint8_avx2 # -- Begin function transpose_int64_uint8_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_uint8_avx2,@function
+transpose_int64_uint8_avx2: # @transpose_int64_uint8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB7_1
+ .p2align 4, 0x90
+.LBB7_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB7_5
+.LBB7_1:
+ test edx, edx
+ jle .LBB7_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB7_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB7_3
+.LBB7_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end7:
+ .size transpose_int64_uint8_avx2, .Lfunc_end7-transpose_int64_uint8_avx2
+ # -- End function
+ .globl transpose_uint8_int8_avx2 # -- Begin function transpose_uint8_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_int8_avx2,@function
+transpose_uint8_int8_avx2: # @transpose_uint8_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB8_1
+ .p2align 4, 0x90
+.LBB8_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB8_5
+.LBB8_1:
+ test edx, edx
+ jle .LBB8_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB8_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB8_3
+.LBB8_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end8:
+ .size transpose_uint8_int8_avx2, .Lfunc_end8-transpose_uint8_int8_avx2
+ # -- End function
+ .globl transpose_int8_int8_avx2 # -- Begin function transpose_int8_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_int8_avx2,@function
+transpose_int8_int8_avx2: # @transpose_int8_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB9_1
+ .p2align 4, 0x90
+.LBB9_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB9_5
+.LBB9_1:
+ test edx, edx
+ jle .LBB9_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB9_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB9_3
+.LBB9_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end9:
+ .size transpose_int8_int8_avx2, .Lfunc_end9-transpose_int8_int8_avx2
+ # -- End function
+ .globl transpose_uint16_int8_avx2 # -- Begin function transpose_uint16_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_int8_avx2,@function
+transpose_uint16_int8_avx2: # @transpose_uint16_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB10_1
+ .p2align 4, 0x90
+.LBB10_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB10_5
+.LBB10_1:
+ test edx, edx
+ jle .LBB10_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB10_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB10_3
+.LBB10_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end10:
+ .size transpose_uint16_int8_avx2, .Lfunc_end10-transpose_uint16_int8_avx2
+ # -- End function
+ .globl transpose_int16_int8_avx2 # -- Begin function transpose_int16_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_int8_avx2,@function
+transpose_int16_int8_avx2: # @transpose_int16_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB11_1
+ .p2align 4, 0x90
+.LBB11_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB11_5
+.LBB11_1:
+ test edx, edx
+ jle .LBB11_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB11_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB11_3
+.LBB11_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end11:
+ .size transpose_int16_int8_avx2, .Lfunc_end11-transpose_int16_int8_avx2
+ # -- End function
+ .globl transpose_uint32_int8_avx2 # -- Begin function transpose_uint32_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_int8_avx2,@function
+transpose_uint32_int8_avx2: # @transpose_uint32_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB12_1
+ .p2align 4, 0x90
+.LBB12_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB12_5
+.LBB12_1:
+ test edx, edx
+ jle .LBB12_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB12_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB12_3
+.LBB12_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end12:
+ .size transpose_uint32_int8_avx2, .Lfunc_end12-transpose_uint32_int8_avx2
+ # -- End function
+ .globl transpose_int32_int8_avx2 # -- Begin function transpose_int32_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_int8_avx2,@function
+transpose_int32_int8_avx2: # @transpose_int32_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB13_1
+ .p2align 4, 0x90
+.LBB13_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB13_5
+.LBB13_1:
+ test edx, edx
+ jle .LBB13_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB13_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB13_3
+.LBB13_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end13:
+ .size transpose_int32_int8_avx2, .Lfunc_end13-transpose_int32_int8_avx2
+ # -- End function
+ .globl transpose_uint64_int8_avx2 # -- Begin function transpose_uint64_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_int8_avx2,@function
+transpose_uint64_int8_avx2: # @transpose_uint64_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB14_1
+ .p2align 4, 0x90
+.LBB14_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB14_5
+.LBB14_1:
+ test edx, edx
+ jle .LBB14_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB14_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB14_3
+.LBB14_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end14:
+ .size transpose_uint64_int8_avx2, .Lfunc_end14-transpose_uint64_int8_avx2
+ # -- End function
+ .globl transpose_int64_int8_avx2 # -- Begin function transpose_int64_int8_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_int8_avx2,@function
+transpose_int64_int8_avx2: # @transpose_int64_int8_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB15_1
+ .p2align 4, 0x90
+.LBB15_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB15_5
+.LBB15_1:
+ test edx, edx
+ jle .LBB15_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB15_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB15_3
+.LBB15_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end15:
+ .size transpose_int64_int8_avx2, .Lfunc_end15-transpose_int64_int8_avx2
+ # -- End function
+ .globl transpose_uint8_uint16_avx2 # -- Begin function transpose_uint8_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_uint16_avx2,@function
+transpose_uint8_uint16_avx2: # @transpose_uint8_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB16_1
+ .p2align 4, 0x90
+.LBB16_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB16_5
+.LBB16_1:
+ test edx, edx
+ jle .LBB16_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB16_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB16_3
+.LBB16_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end16:
+ .size transpose_uint8_uint16_avx2, .Lfunc_end16-transpose_uint8_uint16_avx2
+ # -- End function
+ .globl transpose_int8_uint16_avx2 # -- Begin function transpose_int8_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_uint16_avx2,@function
+transpose_int8_uint16_avx2: # @transpose_int8_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB17_1
+ .p2align 4, 0x90
+.LBB17_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB17_5
+.LBB17_1:
+ test edx, edx
+ jle .LBB17_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB17_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB17_3
+.LBB17_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end17:
+ .size transpose_int8_uint16_avx2, .Lfunc_end17-transpose_int8_uint16_avx2
+ # -- End function
+ .globl transpose_uint16_uint16_avx2 # -- Begin function transpose_uint16_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_uint16_avx2,@function
+transpose_uint16_uint16_avx2: # @transpose_uint16_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB18_1
+ .p2align 4, 0x90
+.LBB18_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB18_5
+.LBB18_1:
+ test edx, edx
+ jle .LBB18_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB18_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB18_3
+.LBB18_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end18:
+ .size transpose_uint16_uint16_avx2, .Lfunc_end18-transpose_uint16_uint16_avx2
+ # -- End function
+ .globl transpose_int16_uint16_avx2 # -- Begin function transpose_int16_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_uint16_avx2,@function
+transpose_int16_uint16_avx2: # @transpose_int16_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB19_1
+ .p2align 4, 0x90
+.LBB19_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB19_5
+.LBB19_1:
+ test edx, edx
+ jle .LBB19_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB19_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB19_3
+.LBB19_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end19:
+ .size transpose_int16_uint16_avx2, .Lfunc_end19-transpose_int16_uint16_avx2
+ # -- End function
+ .globl transpose_uint32_uint16_avx2 # -- Begin function transpose_uint32_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_uint16_avx2,@function
+transpose_uint32_uint16_avx2: # @transpose_uint32_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB20_1
+ .p2align 4, 0x90
+.LBB20_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB20_5
+.LBB20_1:
+ test edx, edx
+ jle .LBB20_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB20_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB20_3
+.LBB20_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end20:
+ .size transpose_uint32_uint16_avx2, .Lfunc_end20-transpose_uint32_uint16_avx2
+ # -- End function
+ .globl transpose_int32_uint16_avx2 # -- Begin function transpose_int32_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_uint16_avx2,@function
+transpose_int32_uint16_avx2: # @transpose_int32_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB21_1
+ .p2align 4, 0x90
+.LBB21_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB21_5
+.LBB21_1:
+ test edx, edx
+ jle .LBB21_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB21_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB21_3
+.LBB21_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end21:
+ .size transpose_int32_uint16_avx2, .Lfunc_end21-transpose_int32_uint16_avx2
+ # -- End function
+ .globl transpose_uint64_uint16_avx2 # -- Begin function transpose_uint64_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_uint16_avx2,@function
+transpose_uint64_uint16_avx2: # @transpose_uint64_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB22_1
+ .p2align 4, 0x90
+.LBB22_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB22_5
+.LBB22_1:
+ test edx, edx
+ jle .LBB22_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB22_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB22_3
+.LBB22_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end22:
+ .size transpose_uint64_uint16_avx2, .Lfunc_end22-transpose_uint64_uint16_avx2
+ # -- End function
+ .globl transpose_int64_uint16_avx2 # -- Begin function transpose_int64_uint16_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_uint16_avx2,@function
+transpose_int64_uint16_avx2: # @transpose_int64_uint16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB23_1
+ .p2align 4, 0x90
+.LBB23_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB23_5
+.LBB23_1:
+ test edx, edx
+ jle .LBB23_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB23_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB23_3
+.LBB23_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end23:
+ .size transpose_int64_uint16_avx2, .Lfunc_end23-transpose_int64_uint16_avx2
+ # -- End function
+ .globl transpose_uint8_int16_avx2 # -- Begin function transpose_uint8_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_int16_avx2,@function
+transpose_uint8_int16_avx2: # @transpose_uint8_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB24_1
+ .p2align 4, 0x90
+.LBB24_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB24_5
+.LBB24_1:
+ test edx, edx
+ jle .LBB24_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB24_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB24_3
+.LBB24_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end24:
+ .size transpose_uint8_int16_avx2, .Lfunc_end24-transpose_uint8_int16_avx2
+ # -- End function
+ .globl transpose_int8_int16_avx2 # -- Begin function transpose_int8_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_int16_avx2,@function
+transpose_int8_int16_avx2: # @transpose_int8_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB25_1
+ .p2align 4, 0x90
+.LBB25_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB25_5
+.LBB25_1:
+ test edx, edx
+ jle .LBB25_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB25_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB25_3
+.LBB25_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end25:
+ .size transpose_int8_int16_avx2, .Lfunc_end25-transpose_int8_int16_avx2
+ # -- End function
+ .globl transpose_uint16_int16_avx2 # -- Begin function transpose_uint16_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_int16_avx2,@function
+transpose_uint16_int16_avx2: # @transpose_uint16_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB26_1
+ .p2align 4, 0x90
+.LBB26_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB26_5
+.LBB26_1:
+ test edx, edx
+ jle .LBB26_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB26_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB26_3
+.LBB26_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end26:
+ .size transpose_uint16_int16_avx2, .Lfunc_end26-transpose_uint16_int16_avx2
+ # -- End function
+ .globl transpose_int16_int16_avx2 # -- Begin function transpose_int16_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_int16_avx2,@function
+transpose_int16_int16_avx2: # @transpose_int16_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB27_1
+ .p2align 4, 0x90
+.LBB27_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB27_5
+.LBB27_1:
+ test edx, edx
+ jle .LBB27_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB27_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB27_3
+.LBB27_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end27:
+ .size transpose_int16_int16_avx2, .Lfunc_end27-transpose_int16_int16_avx2
+ # -- End function
+ .globl transpose_uint32_int16_avx2 # -- Begin function transpose_uint32_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_int16_avx2,@function
+transpose_uint32_int16_avx2: # @transpose_uint32_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB28_1
+ .p2align 4, 0x90
+.LBB28_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB28_5
+.LBB28_1:
+ test edx, edx
+ jle .LBB28_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB28_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB28_3
+.LBB28_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end28:
+ .size transpose_uint32_int16_avx2, .Lfunc_end28-transpose_uint32_int16_avx2
+ # -- End function
+ .globl transpose_int32_int16_avx2 # -- Begin function transpose_int32_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_int16_avx2,@function
+transpose_int32_int16_avx2: # @transpose_int32_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB29_1
+ .p2align 4, 0x90
+.LBB29_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB29_5
+.LBB29_1:
+ test edx, edx
+ jle .LBB29_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB29_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB29_3
+.LBB29_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end29:
+ .size transpose_int32_int16_avx2, .Lfunc_end29-transpose_int32_int16_avx2
+ # -- End function
+ .globl transpose_uint64_int16_avx2 # -- Begin function transpose_uint64_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_int16_avx2,@function
+transpose_uint64_int16_avx2: # @transpose_uint64_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB30_1
+ .p2align 4, 0x90
+.LBB30_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB30_5
+.LBB30_1:
+ test edx, edx
+ jle .LBB30_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB30_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB30_3
+.LBB30_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end30:
+ .size transpose_uint64_int16_avx2, .Lfunc_end30-transpose_uint64_int16_avx2
+ # -- End function
+ .globl transpose_int64_int16_avx2 # -- Begin function transpose_int64_int16_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_int16_avx2,@function
+transpose_int64_int16_avx2: # @transpose_int64_int16_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB31_1
+ .p2align 4, 0x90
+.LBB31_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB31_5
+.LBB31_1:
+ test edx, edx
+ jle .LBB31_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB31_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB31_3
+.LBB31_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end31:
+ .size transpose_int64_int16_avx2, .Lfunc_end31-transpose_int64_int16_avx2
+ # -- End function
+ .globl transpose_uint8_uint32_avx2 # -- Begin function transpose_uint8_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_uint32_avx2,@function
+transpose_uint8_uint32_avx2: # @transpose_uint8_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB32_1
+ .p2align 4, 0x90
+.LBB32_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB32_5
+.LBB32_1:
+ test edx, edx
+ jle .LBB32_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB32_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB32_3
+.LBB32_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end32:
+ .size transpose_uint8_uint32_avx2, .Lfunc_end32-transpose_uint8_uint32_avx2
+ # -- End function
+ .globl transpose_int8_uint32_avx2 # -- Begin function transpose_int8_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_uint32_avx2,@function
+transpose_int8_uint32_avx2: # @transpose_int8_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB33_1
+ .p2align 4, 0x90
+.LBB33_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB33_5
+.LBB33_1:
+ test edx, edx
+ jle .LBB33_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB33_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB33_3
+.LBB33_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end33:
+ .size transpose_int8_uint32_avx2, .Lfunc_end33-transpose_int8_uint32_avx2
+ # -- End function
+ .globl transpose_uint16_uint32_avx2 # -- Begin function transpose_uint16_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_uint32_avx2,@function
+transpose_uint16_uint32_avx2: # @transpose_uint16_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB34_1
+ .p2align 4, 0x90
+.LBB34_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB34_5
+.LBB34_1:
+ test edx, edx
+ jle .LBB34_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB34_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB34_3
+.LBB34_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end34:
+ .size transpose_uint16_uint32_avx2, .Lfunc_end34-transpose_uint16_uint32_avx2
+ # -- End function
+ .globl transpose_int16_uint32_avx2 # -- Begin function transpose_int16_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_uint32_avx2,@function
+transpose_int16_uint32_avx2: # @transpose_int16_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB35_1
+ .p2align 4, 0x90
+.LBB35_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB35_5
+.LBB35_1:
+ test edx, edx
+ jle .LBB35_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB35_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB35_3
+.LBB35_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end35:
+ .size transpose_int16_uint32_avx2, .Lfunc_end35-transpose_int16_uint32_avx2
+ # -- End function
+ .globl transpose_uint32_uint32_avx2 # -- Begin function transpose_uint32_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_uint32_avx2,@function
+transpose_uint32_uint32_avx2: # @transpose_uint32_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB36_1
+ .p2align 4, 0x90
+.LBB36_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov edx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov edx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov edx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB36_5
+.LBB36_1:
+ test edx, edx
+ jle .LBB36_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB36_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB36_3
+.LBB36_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end36:
+ .size transpose_uint32_uint32_avx2, .Lfunc_end36-transpose_uint32_uint32_avx2
+ # -- End function
+ .globl transpose_int32_uint32_avx2 # -- Begin function transpose_int32_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_uint32_avx2,@function
+transpose_int32_uint32_avx2: # @transpose_int32_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB37_1
+ .p2align 4, 0x90
+.LBB37_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsxd rdx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsxd rdx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsxd rdx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB37_5
+.LBB37_1:
+ test edx, edx
+ jle .LBB37_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB37_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB37_3
+.LBB37_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end37:
+ .size transpose_int32_uint32_avx2, .Lfunc_end37-transpose_int32_uint32_avx2
+ # -- End function
+ .globl transpose_uint64_uint32_avx2 # -- Begin function transpose_uint64_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_uint32_avx2,@function
+transpose_uint64_uint32_avx2: # @transpose_uint64_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB38_1
+ .p2align 4, 0x90
+.LBB38_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB38_5
+.LBB38_1:
+ test edx, edx
+ jle .LBB38_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB38_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB38_3
+.LBB38_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end38:
+ .size transpose_uint64_uint32_avx2, .Lfunc_end38-transpose_uint64_uint32_avx2
+ # -- End function
+ .globl transpose_int64_uint32_avx2 # -- Begin function transpose_int64_uint32_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_uint32_avx2,@function
+transpose_int64_uint32_avx2: # @transpose_int64_uint32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB39_1
+ .p2align 4, 0x90
+.LBB39_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB39_5
+.LBB39_1:
+ test edx, edx
+ jle .LBB39_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB39_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB39_3
+.LBB39_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end39:
+ .size transpose_int64_uint32_avx2, .Lfunc_end39-transpose_int64_uint32_avx2
+ # -- End function
+ .globl transpose_uint8_int32_avx2 # -- Begin function transpose_uint8_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_int32_avx2,@function
+transpose_uint8_int32_avx2: # @transpose_uint8_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB40_1
+ .p2align 4, 0x90
+.LBB40_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB40_5
+.LBB40_1:
+ test edx, edx
+ jle .LBB40_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB40_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB40_3
+.LBB40_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end40:
+ .size transpose_uint8_int32_avx2, .Lfunc_end40-transpose_uint8_int32_avx2
+ # -- End function
+ .globl transpose_int8_int32_avx2 # -- Begin function transpose_int8_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_int32_avx2,@function
+transpose_int8_int32_avx2: # @transpose_int8_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB41_1
+ .p2align 4, 0x90
+.LBB41_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB41_5
+.LBB41_1:
+ test edx, edx
+ jle .LBB41_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB41_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB41_3
+.LBB41_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end41:
+ .size transpose_int8_int32_avx2, .Lfunc_end41-transpose_int8_int32_avx2
+ # -- End function
+ .globl transpose_uint16_int32_avx2 # -- Begin function transpose_uint16_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_int32_avx2,@function
+transpose_uint16_int32_avx2: # @transpose_uint16_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB42_1
+ .p2align 4, 0x90
+.LBB42_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB42_5
+.LBB42_1:
+ test edx, edx
+ jle .LBB42_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB42_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB42_3
+.LBB42_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end42:
+ .size transpose_uint16_int32_avx2, .Lfunc_end42-transpose_uint16_int32_avx2
+ # -- End function
+ .globl transpose_int16_int32_avx2 # -- Begin function transpose_int16_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_int32_avx2,@function
+transpose_int16_int32_avx2: # @transpose_int16_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB43_1
+ .p2align 4, 0x90
+.LBB43_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB43_5
+.LBB43_1:
+ test edx, edx
+ jle .LBB43_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB43_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB43_3
+.LBB43_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end43:
+ .size transpose_int16_int32_avx2, .Lfunc_end43-transpose_int16_int32_avx2
+ # -- End function
+ .globl transpose_uint32_int32_avx2 # -- Begin function transpose_uint32_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_int32_avx2,@function
+transpose_uint32_int32_avx2: # @transpose_uint32_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB44_1
+ .p2align 4, 0x90
+.LBB44_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov edx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov edx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov edx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB44_5
+.LBB44_1:
+ test edx, edx
+ jle .LBB44_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB44_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB44_3
+.LBB44_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end44:
+ .size transpose_uint32_int32_avx2, .Lfunc_end44-transpose_uint32_int32_avx2
+ # -- End function
+ .globl transpose_int32_int32_avx2 # -- Begin function transpose_int32_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_int32_avx2,@function
+transpose_int32_int32_avx2: # @transpose_int32_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB45_1
+ .p2align 4, 0x90
+.LBB45_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsxd rdx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsxd rdx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsxd rdx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB45_5
+.LBB45_1:
+ test edx, edx
+ jle .LBB45_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB45_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB45_3
+.LBB45_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end45:
+ .size transpose_int32_int32_avx2, .Lfunc_end45-transpose_int32_int32_avx2
+ # -- End function
+ .globl transpose_uint64_int32_avx2 # -- Begin function transpose_uint64_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_int32_avx2,@function
+transpose_uint64_int32_avx2: # @transpose_uint64_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB46_1
+ .p2align 4, 0x90
+.LBB46_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB46_5
+.LBB46_1:
+ test edx, edx
+ jle .LBB46_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB46_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB46_3
+.LBB46_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end46:
+ .size transpose_uint64_int32_avx2, .Lfunc_end46-transpose_uint64_int32_avx2
+ # -- End function
+ .globl transpose_int64_int32_avx2 # -- Begin function transpose_int64_int32_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_int32_avx2,@function
+transpose_int64_int32_avx2: # @transpose_int64_int32_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB47_1
+ .p2align 4, 0x90
+.LBB47_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB47_5
+.LBB47_1:
+ test edx, edx
+ jle .LBB47_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB47_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB47_3
+.LBB47_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end47:
+ .size transpose_int64_int32_avx2, .Lfunc_end47-transpose_int64_int32_avx2
+ # -- End function
+ .globl transpose_uint8_uint64_avx2 # -- Begin function transpose_uint8_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_uint64_avx2,@function
+transpose_uint8_uint64_avx2: # @transpose_uint8_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB48_1
+ .p2align 4, 0x90
+.LBB48_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB48_5
+.LBB48_1:
+ test edx, edx
+ jle .LBB48_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB48_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB48_3
+.LBB48_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end48:
+ .size transpose_uint8_uint64_avx2, .Lfunc_end48-transpose_uint8_uint64_avx2
+ # -- End function
+ .globl transpose_int8_uint64_avx2 # -- Begin function transpose_int8_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_uint64_avx2,@function
+transpose_int8_uint64_avx2: # @transpose_int8_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB49_1
+ .p2align 4, 0x90
+.LBB49_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB49_5
+.LBB49_1:
+ test edx, edx
+ jle .LBB49_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB49_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB49_3
+.LBB49_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end49:
+ .size transpose_int8_uint64_avx2, .Lfunc_end49-transpose_int8_uint64_avx2
+ # -- End function
+ .globl transpose_uint16_uint64_avx2 # -- Begin function transpose_uint16_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_uint64_avx2,@function
+transpose_uint16_uint64_avx2: # @transpose_uint16_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB50_1
+ .p2align 4, 0x90
+.LBB50_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB50_5
+.LBB50_1:
+ test edx, edx
+ jle .LBB50_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB50_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB50_3
+.LBB50_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end50:
+ .size transpose_uint16_uint64_avx2, .Lfunc_end50-transpose_uint16_uint64_avx2
+ # -- End function
+ .globl transpose_int16_uint64_avx2 # -- Begin function transpose_int16_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_uint64_avx2,@function
+transpose_int16_uint64_avx2: # @transpose_int16_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB51_1
+ .p2align 4, 0x90
+.LBB51_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB51_5
+.LBB51_1:
+ test edx, edx
+ jle .LBB51_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB51_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB51_3
+.LBB51_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end51:
+ .size transpose_int16_uint64_avx2, .Lfunc_end51-transpose_int16_uint64_avx2
+ # -- End function
+ .globl transpose_uint32_uint64_avx2 # -- Begin function transpose_uint32_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_uint64_avx2,@function
+transpose_uint32_uint64_avx2: # @transpose_uint32_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB52_1
+ .p2align 4, 0x90
+.LBB52_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov edx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov edx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov edx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB52_5
+.LBB52_1:
+ test edx, edx
+ jle .LBB52_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB52_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB52_3
+.LBB52_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end52:
+ .size transpose_uint32_uint64_avx2, .Lfunc_end52-transpose_uint32_uint64_avx2
+ # -- End function
+ .globl transpose_int32_uint64_avx2 # -- Begin function transpose_int32_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_uint64_avx2,@function
+transpose_int32_uint64_avx2: # @transpose_int32_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB53_1
+ .p2align 4, 0x90
+.LBB53_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsxd rdx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsxd rdx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsxd rdx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB53_5
+.LBB53_1:
+ test edx, edx
+ jle .LBB53_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB53_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB53_3
+.LBB53_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end53:
+ .size transpose_int32_uint64_avx2, .Lfunc_end53-transpose_int32_uint64_avx2
+ # -- End function
+ .globl transpose_uint64_uint64_avx2 # -- Begin function transpose_uint64_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_uint64_avx2,@function
+transpose_uint64_uint64_avx2: # @transpose_uint64_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB54_1
+ .p2align 4, 0x90
+.LBB54_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB54_5
+.LBB54_1:
+ test edx, edx
+ jle .LBB54_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB54_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB54_3
+.LBB54_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end54:
+ .size transpose_uint64_uint64_avx2, .Lfunc_end54-transpose_uint64_uint64_avx2
+ # -- End function
+ .globl transpose_int64_uint64_avx2 # -- Begin function transpose_int64_uint64_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_uint64_avx2,@function
+transpose_int64_uint64_avx2: # @transpose_int64_uint64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB55_1
+ .p2align 4, 0x90
+.LBB55_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB55_5
+.LBB55_1:
+ test edx, edx
+ jle .LBB55_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB55_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB55_3
+.LBB55_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end55:
+ .size transpose_int64_uint64_avx2, .Lfunc_end55-transpose_int64_uint64_avx2
+ # -- End function
+ .globl transpose_uint8_int64_avx2 # -- Begin function transpose_uint8_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint8_int64_avx2,@function
+transpose_uint8_int64_avx2: # @transpose_uint8_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB56_1
+ .p2align 4, 0x90
+.LBB56_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB56_5
+.LBB56_1:
+ test edx, edx
+ jle .LBB56_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB56_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB56_3
+.LBB56_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end56:
+ .size transpose_uint8_int64_avx2, .Lfunc_end56-transpose_uint8_int64_avx2
+ # -- End function
+ .globl transpose_int8_int64_avx2 # -- Begin function transpose_int8_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_int8_int64_avx2,@function
+transpose_int8_int64_avx2: # @transpose_int8_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB57_1
+ .p2align 4, 0x90
+.LBB57_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB57_5
+.LBB57_1:
+ test edx, edx
+ jle .LBB57_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB57_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB57_3
+.LBB57_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end57:
+ .size transpose_int8_int64_avx2, .Lfunc_end57-transpose_int8_int64_avx2
+ # -- End function
+ .globl transpose_uint16_int64_avx2 # -- Begin function transpose_uint16_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint16_int64_avx2,@function
+transpose_uint16_int64_avx2: # @transpose_uint16_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB58_1
+ .p2align 4, 0x90
+.LBB58_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB58_5
+.LBB58_1:
+ test edx, edx
+ jle .LBB58_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB58_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB58_3
+.LBB58_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end58:
+ .size transpose_uint16_int64_avx2, .Lfunc_end58-transpose_uint16_int64_avx2
+ # -- End function
+ .globl transpose_int16_int64_avx2 # -- Begin function transpose_int16_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_int16_int64_avx2,@function
+transpose_int16_int64_avx2: # @transpose_int16_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB59_1
+ .p2align 4, 0x90
+.LBB59_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB59_5
+.LBB59_1:
+ test edx, edx
+ jle .LBB59_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB59_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB59_3
+.LBB59_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end59:
+ .size transpose_int16_int64_avx2, .Lfunc_end59-transpose_int16_int64_avx2
+ # -- End function
+ .globl transpose_uint32_int64_avx2 # -- Begin function transpose_uint32_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint32_int64_avx2,@function
+transpose_uint32_int64_avx2: # @transpose_uint32_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB60_1
+ .p2align 4, 0x90
+.LBB60_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov edx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov edx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov edx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB60_5
+.LBB60_1:
+ test edx, edx
+ jle .LBB60_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB60_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB60_3
+.LBB60_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end60:
+ .size transpose_uint32_int64_avx2, .Lfunc_end60-transpose_uint32_int64_avx2
+ # -- End function
+ .globl transpose_int32_int64_avx2 # -- Begin function transpose_int32_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_int32_int64_avx2,@function
+transpose_int32_int64_avx2: # @transpose_int32_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB61_1
+ .p2align 4, 0x90
+.LBB61_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsxd rdx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsxd rdx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsxd rdx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB61_5
+.LBB61_1:
+ test edx, edx
+ jle .LBB61_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB61_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB61_3
+.LBB61_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end61:
+ .size transpose_int32_int64_avx2, .Lfunc_end61-transpose_int32_int64_avx2
+ # -- End function
+ .globl transpose_uint64_int64_avx2 # -- Begin function transpose_uint64_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_uint64_int64_avx2,@function
+transpose_uint64_int64_avx2: # @transpose_uint64_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB62_1
+ .p2align 4, 0x90
+.LBB62_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB62_5
+.LBB62_1:
+ test edx, edx
+ jle .LBB62_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB62_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB62_3
+.LBB62_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end62:
+ .size transpose_uint64_int64_avx2, .Lfunc_end62-transpose_uint64_int64_avx2
+ # -- End function
+ .globl transpose_int64_int64_avx2 # -- Begin function transpose_int64_int64_avx2
+ .p2align 4, 0x90
+ .type transpose_int64_int64_avx2,@function
+transpose_int64_int64_avx2: # @transpose_int64_int64_avx2
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB63_1
+ .p2align 4, 0x90
+.LBB63_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB63_5
+.LBB63_1:
+ test edx, edx
+ jle .LBB63_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB63_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB63_3
+.LBB63_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end63:
+ .size transpose_int64_int64_avx2, .Lfunc_end63-transpose_int64_int64_avx2
+ # -- End function
+ .ident "Ubuntu clang version 11.0.0-2~ubuntu20.04.1"
+ .section ".note.GNU-stack","",@progbits
+ .addrsig
diff --git a/go/internal/utils/_lib/transpose_ints_sse4_amd64.s b/go/internal/utils/_lib/transpose_ints_sse4_amd64.s
new file mode 100644
index 0000000000..c177d4478c
--- /dev/null
+++ b/go/internal/utils/_lib/transpose_ints_sse4_amd64.s
@@ -0,0 +1,3334 @@
+ .text
+ .intel_syntax noprefix
+ .file "transpose_ints.c"
+ .globl transpose_uint8_uint8_sse4 # -- Begin function transpose_uint8_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_uint8_sse4,@function
+transpose_uint8_uint8_sse4: # @transpose_uint8_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB0_1
+ .p2align 4, 0x90
+.LBB0_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB0_5
+.LBB0_1:
+ test edx, edx
+ jle .LBB0_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB0_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB0_3
+.LBB0_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end0:
+ .size transpose_uint8_uint8_sse4, .Lfunc_end0-transpose_uint8_uint8_sse4
+ # -- End function
+ .globl transpose_int8_uint8_sse4 # -- Begin function transpose_int8_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_uint8_sse4,@function
+transpose_int8_uint8_sse4: # @transpose_int8_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB1_1
+ .p2align 4, 0x90
+.LBB1_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB1_5
+.LBB1_1:
+ test edx, edx
+ jle .LBB1_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB1_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB1_3
+.LBB1_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end1:
+ .size transpose_int8_uint8_sse4, .Lfunc_end1-transpose_int8_uint8_sse4
+ # -- End function
+ .globl transpose_uint16_uint8_sse4 # -- Begin function transpose_uint16_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_uint8_sse4,@function
+transpose_uint16_uint8_sse4: # @transpose_uint16_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB2_1
+ .p2align 4, 0x90
+.LBB2_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB2_5
+.LBB2_1:
+ test edx, edx
+ jle .LBB2_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB2_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB2_3
+.LBB2_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end2:
+ .size transpose_uint16_uint8_sse4, .Lfunc_end2-transpose_uint16_uint8_sse4
+ # -- End function
+ .globl transpose_int16_uint8_sse4 # -- Begin function transpose_int16_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_uint8_sse4,@function
+transpose_int16_uint8_sse4: # @transpose_int16_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB3_1
+ .p2align 4, 0x90
+.LBB3_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB3_5
+.LBB3_1:
+ test edx, edx
+ jle .LBB3_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB3_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB3_3
+.LBB3_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end3:
+ .size transpose_int16_uint8_sse4, .Lfunc_end3-transpose_int16_uint8_sse4
+ # -- End function
+ .globl transpose_uint32_uint8_sse4 # -- Begin function transpose_uint32_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_uint8_sse4,@function
+transpose_uint32_uint8_sse4: # @transpose_uint32_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB4_1
+ .p2align 4, 0x90
+.LBB4_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB4_5
+.LBB4_1:
+ test edx, edx
+ jle .LBB4_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB4_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB4_3
+.LBB4_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end4:
+ .size transpose_uint32_uint8_sse4, .Lfunc_end4-transpose_uint32_uint8_sse4
+ # -- End function
+ .globl transpose_int32_uint8_sse4 # -- Begin function transpose_int32_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_uint8_sse4,@function
+transpose_int32_uint8_sse4: # @transpose_int32_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB5_1
+ .p2align 4, 0x90
+.LBB5_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB5_5
+.LBB5_1:
+ test edx, edx
+ jle .LBB5_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB5_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB5_3
+.LBB5_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end5:
+ .size transpose_int32_uint8_sse4, .Lfunc_end5-transpose_int32_uint8_sse4
+ # -- End function
+ .globl transpose_uint64_uint8_sse4 # -- Begin function transpose_uint64_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_uint8_sse4,@function
+transpose_uint64_uint8_sse4: # @transpose_uint64_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB6_1
+ .p2align 4, 0x90
+.LBB6_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB6_5
+.LBB6_1:
+ test edx, edx
+ jle .LBB6_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB6_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB6_3
+.LBB6_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end6:
+ .size transpose_uint64_uint8_sse4, .Lfunc_end6-transpose_uint64_uint8_sse4
+ # -- End function
+ .globl transpose_int64_uint8_sse4 # -- Begin function transpose_int64_uint8_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_uint8_sse4,@function
+transpose_int64_uint8_sse4: # @transpose_int64_uint8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB7_1
+ .p2align 4, 0x90
+.LBB7_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB7_5
+.LBB7_1:
+ test edx, edx
+ jle .LBB7_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB7_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB7_3
+.LBB7_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end7:
+ .size transpose_int64_uint8_sse4, .Lfunc_end7-transpose_int64_uint8_sse4
+ # -- End function
+ .globl transpose_uint8_int8_sse4 # -- Begin function transpose_uint8_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_int8_sse4,@function
+transpose_uint8_int8_sse4: # @transpose_uint8_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB8_1
+ .p2align 4, 0x90
+.LBB8_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB8_5
+.LBB8_1:
+ test edx, edx
+ jle .LBB8_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB8_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB8_3
+.LBB8_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end8:
+ .size transpose_uint8_int8_sse4, .Lfunc_end8-transpose_uint8_int8_sse4
+ # -- End function
+ .globl transpose_int8_int8_sse4 # -- Begin function transpose_int8_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_int8_sse4,@function
+transpose_int8_int8_sse4: # @transpose_int8_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB9_1
+ .p2align 4, 0x90
+.LBB9_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB9_5
+.LBB9_1:
+ test edx, edx
+ jle .LBB9_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB9_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB9_3
+.LBB9_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end9:
+ .size transpose_int8_int8_sse4, .Lfunc_end9-transpose_int8_int8_sse4
+ # -- End function
+ .globl transpose_uint16_int8_sse4 # -- Begin function transpose_uint16_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_int8_sse4,@function
+transpose_uint16_int8_sse4: # @transpose_uint16_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB10_1
+ .p2align 4, 0x90
+.LBB10_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB10_5
+.LBB10_1:
+ test edx, edx
+ jle .LBB10_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB10_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB10_3
+.LBB10_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end10:
+ .size transpose_uint16_int8_sse4, .Lfunc_end10-transpose_uint16_int8_sse4
+ # -- End function
+ .globl transpose_int16_int8_sse4 # -- Begin function transpose_int16_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_int8_sse4,@function
+transpose_int16_int8_sse4: # @transpose_int16_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB11_1
+ .p2align 4, 0x90
+.LBB11_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB11_5
+.LBB11_1:
+ test edx, edx
+ jle .LBB11_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB11_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + 2*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB11_3
+.LBB11_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end11:
+ .size transpose_int16_int8_sse4, .Lfunc_end11-transpose_int16_int8_sse4
+ # -- End function
+ .globl transpose_uint32_int8_sse4 # -- Begin function transpose_uint32_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_int8_sse4,@function
+transpose_uint32_int8_sse4: # @transpose_uint32_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB12_1
+ .p2align 4, 0x90
+.LBB12_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB12_5
+.LBB12_1:
+ test edx, edx
+ jle .LBB12_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB12_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB12_3
+.LBB12_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end12:
+ .size transpose_uint32_int8_sse4, .Lfunc_end12-transpose_uint32_int8_sse4
+ # -- End function
+ .globl transpose_int32_int8_sse4 # -- Begin function transpose_int32_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_int8_sse4,@function
+transpose_int32_int8_sse4: # @transpose_int32_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB13_1
+ .p2align 4, 0x90
+.LBB13_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB13_5
+.LBB13_1:
+ test edx, edx
+ jle .LBB13_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB13_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 4*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB13_3
+.LBB13_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end13:
+ .size transpose_int32_int8_sse4, .Lfunc_end13-transpose_int32_int8_sse4
+ # -- End function
+ .globl transpose_uint64_int8_sse4 # -- Begin function transpose_uint64_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_int8_sse4,@function
+transpose_uint64_int8_sse4: # @transpose_uint64_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB14_1
+ .p2align 4, 0x90
+.LBB14_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB14_5
+.LBB14_1:
+ test edx, edx
+ jle .LBB14_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB14_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB14_3
+.LBB14_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end14:
+ .size transpose_uint64_int8_sse4, .Lfunc_end14-transpose_uint64_int8_sse4
+ # -- End function
+ .globl transpose_int64_int8_sse4 # -- Begin function transpose_int64_int8_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_int8_sse4,@function
+transpose_int64_int8_sse4: # @transpose_int64_int8_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB15_1
+ .p2align 4, 0x90
+.LBB15_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi], dl
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 1], dl
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 2], dl
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, byte ptr [rcx + 4*rdx]
+ mov byte ptr [rsi + 3], dl
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 4
+ cmp eax, 7
+ jg .LBB15_5
+.LBB15_1:
+ test edx, edx
+ jle .LBB15_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB15_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 8*r8]
+ movzx eax, byte ptr [rcx + 4*rax]
+ mov byte ptr [rsi + r8], al
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB15_3
+.LBB15_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end15:
+ .size transpose_int64_int8_sse4, .Lfunc_end15-transpose_int64_int8_sse4
+ # -- End function
+ .globl transpose_uint8_uint16_sse4 # -- Begin function transpose_uint8_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_uint16_sse4,@function
+transpose_uint8_uint16_sse4: # @transpose_uint8_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB16_1
+ .p2align 4, 0x90
+.LBB16_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB16_5
+.LBB16_1:
+ test edx, edx
+ jle .LBB16_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB16_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB16_3
+.LBB16_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end16:
+ .size transpose_uint8_uint16_sse4, .Lfunc_end16-transpose_uint8_uint16_sse4
+ # -- End function
+ .globl transpose_int8_uint16_sse4 # -- Begin function transpose_int8_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_uint16_sse4,@function
+transpose_int8_uint16_sse4: # @transpose_int8_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB17_1
+ .p2align 4, 0x90
+.LBB17_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB17_5
+.LBB17_1:
+ test edx, edx
+ jle .LBB17_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB17_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB17_3
+.LBB17_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end17:
+ .size transpose_int8_uint16_sse4, .Lfunc_end17-transpose_int8_uint16_sse4
+ # -- End function
+ .globl transpose_uint16_uint16_sse4 # -- Begin function transpose_uint16_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_uint16_sse4,@function
+transpose_uint16_uint16_sse4: # @transpose_uint16_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB18_1
+ .p2align 4, 0x90
+.LBB18_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB18_5
+.LBB18_1:
+ test edx, edx
+ jle .LBB18_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB18_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB18_3
+.LBB18_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end18:
+ .size transpose_uint16_uint16_sse4, .Lfunc_end18-transpose_uint16_uint16_sse4
+ # -- End function
+ .globl transpose_int16_uint16_sse4 # -- Begin function transpose_int16_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_uint16_sse4,@function
+transpose_int16_uint16_sse4: # @transpose_int16_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB19_1
+ .p2align 4, 0x90
+.LBB19_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB19_5
+.LBB19_1:
+ test edx, edx
+ jle .LBB19_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB19_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB19_3
+.LBB19_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end19:
+ .size transpose_int16_uint16_sse4, .Lfunc_end19-transpose_int16_uint16_sse4
+ # -- End function
+ .globl transpose_uint32_uint16_sse4 # -- Begin function transpose_uint32_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_uint16_sse4,@function
+transpose_uint32_uint16_sse4: # @transpose_uint32_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB20_1
+ .p2align 4, 0x90
+.LBB20_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB20_5
+.LBB20_1:
+ test edx, edx
+ jle .LBB20_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB20_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB20_3
+.LBB20_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end20:
+ .size transpose_uint32_uint16_sse4, .Lfunc_end20-transpose_uint32_uint16_sse4
+ # -- End function
+ .globl transpose_int32_uint16_sse4 # -- Begin function transpose_int32_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_uint16_sse4,@function
+transpose_int32_uint16_sse4: # @transpose_int32_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB21_1
+ .p2align 4, 0x90
+.LBB21_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB21_5
+.LBB21_1:
+ test edx, edx
+ jle .LBB21_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB21_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB21_3
+.LBB21_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end21:
+ .size transpose_int32_uint16_sse4, .Lfunc_end21-transpose_int32_uint16_sse4
+ # -- End function
+ .globl transpose_uint64_uint16_sse4 # -- Begin function transpose_uint64_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_uint16_sse4,@function
+transpose_uint64_uint16_sse4: # @transpose_uint64_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB22_1
+ .p2align 4, 0x90
+.LBB22_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB22_5
+.LBB22_1:
+ test edx, edx
+ jle .LBB22_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB22_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB22_3
+.LBB22_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end22:
+ .size transpose_uint64_uint16_sse4, .Lfunc_end22-transpose_uint64_uint16_sse4
+ # -- End function
+ .globl transpose_int64_uint16_sse4 # -- Begin function transpose_int64_uint16_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_uint16_sse4,@function
+transpose_int64_uint16_sse4: # @transpose_int64_uint16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB23_1
+ .p2align 4, 0x90
+.LBB23_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB23_5
+.LBB23_1:
+ test edx, edx
+ jle .LBB23_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB23_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB23_3
+.LBB23_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end23:
+ .size transpose_int64_uint16_sse4, .Lfunc_end23-transpose_int64_uint16_sse4
+ # -- End function
+ .globl transpose_uint8_int16_sse4 # -- Begin function transpose_uint8_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_int16_sse4,@function
+transpose_uint8_int16_sse4: # @transpose_uint8_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB24_1
+ .p2align 4, 0x90
+.LBB24_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB24_5
+.LBB24_1:
+ test edx, edx
+ jle .LBB24_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB24_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB24_3
+.LBB24_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end24:
+ .size transpose_uint8_int16_sse4, .Lfunc_end24-transpose_uint8_int16_sse4
+ # -- End function
+ .globl transpose_int8_int16_sse4 # -- Begin function transpose_int8_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_int16_sse4,@function
+transpose_int8_int16_sse4: # @transpose_int8_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB25_1
+ .p2align 4, 0x90
+.LBB25_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, byte ptr [rdi + 1]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, byte ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, byte ptr [rdi + 3]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB25_5
+.LBB25_1:
+ test edx, edx
+ jle .LBB25_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB25_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + 2*r8], ax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB25_3
+.LBB25_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end25:
+ .size transpose_int8_int16_sse4, .Lfunc_end25-transpose_int8_int16_sse4
+ # -- End function
+ .globl transpose_uint16_int16_sse4 # -- Begin function transpose_uint16_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_int16_sse4,@function
+transpose_uint16_int16_sse4: # @transpose_uint16_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB26_1
+ .p2align 4, 0x90
+.LBB26_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movzx edx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movzx edx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movzx edx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB26_5
+.LBB26_1:
+ test edx, edx
+ jle .LBB26_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB26_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB26_3
+.LBB26_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end26:
+ .size transpose_uint16_int16_sse4, .Lfunc_end26-transpose_uint16_int16_sse4
+ # -- End function
+ .globl transpose_int16_int16_sse4 # -- Begin function transpose_int16_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_int16_sse4,@function
+transpose_int16_int16_sse4: # @transpose_int16_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB27_1
+ .p2align 4, 0x90
+.LBB27_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsx rdx, word ptr [rdi + 2]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsx rdx, word ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsx rdx, word ptr [rdi + 6]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB27_5
+.LBB27_1:
+ test edx, edx
+ jle .LBB27_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB27_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB27_3
+.LBB27_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end27:
+ .size transpose_int16_int16_sse4, .Lfunc_end27-transpose_int16_int16_sse4
+ # -- End function
+ .globl transpose_uint32_int16_sse4 # -- Begin function transpose_uint32_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_int16_sse4,@function
+transpose_uint32_int16_sse4: # @transpose_uint32_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB28_1
+ .p2align 4, 0x90
+.LBB28_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov edx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov edx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov edx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB28_5
+.LBB28_1:
+ test edx, edx
+ jle .LBB28_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB28_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB28_3
+.LBB28_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end28:
+ .size transpose_uint32_int16_sse4, .Lfunc_end28-transpose_uint32_int16_sse4
+ # -- End function
+ .globl transpose_int32_int16_sse4 # -- Begin function transpose_int32_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_int16_sse4,@function
+transpose_int32_int16_sse4: # @transpose_int32_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB29_1
+ .p2align 4, 0x90
+.LBB29_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ movsxd rdx, dword ptr [rdi + 4]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ movsxd rdx, dword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ movsxd rdx, dword ptr [rdi + 12]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB29_5
+.LBB29_1:
+ test edx, edx
+ jle .LBB29_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB29_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + 2*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB29_3
+.LBB29_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end29:
+ .size transpose_int32_int16_sse4, .Lfunc_end29-transpose_int32_int16_sse4
+ # -- End function
+ .globl transpose_uint64_int16_sse4 # -- Begin function transpose_uint64_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_int16_sse4,@function
+transpose_uint64_int16_sse4: # @transpose_uint64_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB30_1
+ .p2align 4, 0x90
+.LBB30_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB30_5
+.LBB30_1:
+ test edx, edx
+ jle .LBB30_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB30_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB30_3
+.LBB30_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end30:
+ .size transpose_uint64_int16_sse4, .Lfunc_end30-transpose_uint64_int16_sse4
+ # -- End function
+ .globl transpose_int64_int16_sse4 # -- Begin function transpose_int64_int16_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_int16_sse4,@function
+transpose_int64_int16_sse4: # @transpose_int64_int16_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB31_1
+ .p2align 4, 0x90
+.LBB31_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi], dx
+ mov rdx, qword ptr [rdi + 8]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 2], dx
+ mov rdx, qword ptr [rdi + 16]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 4], dx
+ mov rdx, qword ptr [rdi + 24]
+ movzx edx, word ptr [rcx + 4*rdx]
+ mov word ptr [rsi + 6], dx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 8
+ cmp eax, 7
+ jg .LBB31_5
+.LBB31_1:
+ test edx, edx
+ jle .LBB31_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB31_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 4*r8]
+ movzx eax, word ptr [rcx + 4*rax]
+ mov word ptr [rsi + r8], ax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB31_3
+.LBB31_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end31:
+ .size transpose_int64_int16_sse4, .Lfunc_end31-transpose_int64_int16_sse4
+ # -- End function
+ .globl transpose_uint8_uint32_sse4 # -- Begin function transpose_uint8_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_uint32_sse4,@function
+transpose_uint8_uint32_sse4: # @transpose_uint8_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB32_1
+ .p2align 4, 0x90
+.LBB32_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB32_5
+.LBB32_1:
+ test edx, edx
+ jle .LBB32_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB32_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB32_3
+.LBB32_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end32:
+ .size transpose_uint8_uint32_sse4, .Lfunc_end32-transpose_uint8_uint32_sse4
+ # -- End function
+ .globl transpose_int8_uint32_sse4 # -- Begin function transpose_int8_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_uint32_sse4,@function
+transpose_int8_uint32_sse4: # @transpose_int8_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB33_1
+ .p2align 4, 0x90
+.LBB33_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB33_5
+.LBB33_1:
+ test edx, edx
+ jle .LBB33_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB33_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB33_3
+.LBB33_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end33:
+ .size transpose_int8_uint32_sse4, .Lfunc_end33-transpose_int8_uint32_sse4
+ # -- End function
+ .globl transpose_uint16_uint32_sse4 # -- Begin function transpose_uint16_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_uint32_sse4,@function
+transpose_uint16_uint32_sse4: # @transpose_uint16_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB34_1
+ .p2align 4, 0x90
+.LBB34_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB34_5
+.LBB34_1:
+ test edx, edx
+ jle .LBB34_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB34_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB34_3
+.LBB34_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end34:
+ .size transpose_uint16_uint32_sse4, .Lfunc_end34-transpose_uint16_uint32_sse4
+ # -- End function
+ .globl transpose_int16_uint32_sse4 # -- Begin function transpose_int16_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_uint32_sse4,@function
+transpose_int16_uint32_sse4: # @transpose_int16_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB35_1
+ .p2align 4, 0x90
+.LBB35_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB35_5
+.LBB35_1:
+ test edx, edx
+ jle .LBB35_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB35_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB35_3
+.LBB35_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end35:
+ .size transpose_int16_uint32_sse4, .Lfunc_end35-transpose_int16_uint32_sse4
+ # -- End function
+ .globl transpose_uint32_uint32_sse4 # -- Begin function transpose_uint32_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_uint32_sse4,@function
+transpose_uint32_uint32_sse4: # @transpose_uint32_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB36_1
+ .p2align 4, 0x90
+.LBB36_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov edx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov edx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov edx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB36_5
+.LBB36_1:
+ test edx, edx
+ jle .LBB36_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB36_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB36_3
+.LBB36_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end36:
+ .size transpose_uint32_uint32_sse4, .Lfunc_end36-transpose_uint32_uint32_sse4
+ # -- End function
+ .globl transpose_int32_uint32_sse4 # -- Begin function transpose_int32_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_uint32_sse4,@function
+transpose_int32_uint32_sse4: # @transpose_int32_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB37_1
+ .p2align 4, 0x90
+.LBB37_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsxd rdx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsxd rdx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsxd rdx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB37_5
+.LBB37_1:
+ test edx, edx
+ jle .LBB37_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB37_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB37_3
+.LBB37_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end37:
+ .size transpose_int32_uint32_sse4, .Lfunc_end37-transpose_int32_uint32_sse4
+ # -- End function
+ .globl transpose_uint64_uint32_sse4 # -- Begin function transpose_uint64_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_uint32_sse4,@function
+transpose_uint64_uint32_sse4: # @transpose_uint64_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB38_1
+ .p2align 4, 0x90
+.LBB38_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB38_5
+.LBB38_1:
+ test edx, edx
+ jle .LBB38_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB38_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB38_3
+.LBB38_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end38:
+ .size transpose_uint64_uint32_sse4, .Lfunc_end38-transpose_uint64_uint32_sse4
+ # -- End function
+ .globl transpose_int64_uint32_sse4 # -- Begin function transpose_int64_uint32_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_uint32_sse4,@function
+transpose_int64_uint32_sse4: # @transpose_int64_uint32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB39_1
+ .p2align 4, 0x90
+.LBB39_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB39_5
+.LBB39_1:
+ test edx, edx
+ jle .LBB39_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB39_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB39_3
+.LBB39_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end39:
+ .size transpose_int64_uint32_sse4, .Lfunc_end39-transpose_int64_uint32_sse4
+ # -- End function
+ .globl transpose_uint8_int32_sse4 # -- Begin function transpose_uint8_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_int32_sse4,@function
+transpose_uint8_int32_sse4: # @transpose_uint8_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB40_1
+ .p2align 4, 0x90
+.LBB40_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB40_5
+.LBB40_1:
+ test edx, edx
+ jle .LBB40_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB40_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB40_3
+.LBB40_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end40:
+ .size transpose_uint8_int32_sse4, .Lfunc_end40-transpose_uint8_int32_sse4
+ # -- End function
+ .globl transpose_int8_int32_sse4 # -- Begin function transpose_int8_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_int32_sse4,@function
+transpose_int8_int32_sse4: # @transpose_int8_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB41_1
+ .p2align 4, 0x90
+.LBB41_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, byte ptr [rdi + 1]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, byte ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, byte ptr [rdi + 3]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB41_5
+.LBB41_1:
+ test edx, edx
+ jle .LBB41_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB41_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 4*r8], eax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB41_3
+.LBB41_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end41:
+ .size transpose_int8_int32_sse4, .Lfunc_end41-transpose_int8_int32_sse4
+ # -- End function
+ .globl transpose_uint16_int32_sse4 # -- Begin function transpose_uint16_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_int32_sse4,@function
+transpose_uint16_int32_sse4: # @transpose_uint16_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB42_1
+ .p2align 4, 0x90
+.LBB42_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movzx edx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movzx edx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movzx edx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB42_5
+.LBB42_1:
+ test edx, edx
+ jle .LBB42_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB42_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB42_3
+.LBB42_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end42:
+ .size transpose_uint16_int32_sse4, .Lfunc_end42-transpose_uint16_int32_sse4
+ # -- End function
+ .globl transpose_int16_int32_sse4 # -- Begin function transpose_int16_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_int32_sse4,@function
+transpose_int16_int32_sse4: # @transpose_int16_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB43_1
+ .p2align 4, 0x90
+.LBB43_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsx rdx, word ptr [rdi + 2]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsx rdx, word ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsx rdx, word ptr [rdi + 6]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB43_5
+.LBB43_1:
+ test edx, edx
+ jle .LBB43_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB43_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + 2*r8], eax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB43_3
+.LBB43_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end43:
+ .size transpose_int16_int32_sse4, .Lfunc_end43-transpose_int16_int32_sse4
+ # -- End function
+ .globl transpose_uint32_int32_sse4 # -- Begin function transpose_uint32_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_int32_sse4,@function
+transpose_uint32_int32_sse4: # @transpose_uint32_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB44_1
+ .p2align 4, 0x90
+.LBB44_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov edx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov edx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov edx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB44_5
+.LBB44_1:
+ test edx, edx
+ jle .LBB44_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB44_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB44_3
+.LBB44_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end44:
+ .size transpose_uint32_int32_sse4, .Lfunc_end44-transpose_uint32_int32_sse4
+ # -- End function
+ .globl transpose_int32_int32_sse4 # -- Begin function transpose_int32_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_int32_sse4,@function
+transpose_int32_int32_sse4: # @transpose_int32_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB45_1
+ .p2align 4, 0x90
+.LBB45_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ movsxd rdx, dword ptr [rdi + 4]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ movsxd rdx, dword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ movsxd rdx, dword ptr [rdi + 12]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB45_5
+.LBB45_1:
+ test edx, edx
+ jle .LBB45_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB45_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB45_3
+.LBB45_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end45:
+ .size transpose_int32_int32_sse4, .Lfunc_end45-transpose_int32_int32_sse4
+ # -- End function
+ .globl transpose_uint64_int32_sse4 # -- Begin function transpose_uint64_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_int32_sse4,@function
+transpose_uint64_int32_sse4: # @transpose_uint64_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB46_1
+ .p2align 4, 0x90
+.LBB46_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB46_5
+.LBB46_1:
+ test edx, edx
+ jle .LBB46_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB46_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB46_3
+.LBB46_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end46:
+ .size transpose_uint64_int32_sse4, .Lfunc_end46-transpose_uint64_int32_sse4
+ # -- End function
+ .globl transpose_int64_int32_sse4 # -- Begin function transpose_int64_int32_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_int32_sse4,@function
+transpose_int64_int32_sse4: # @transpose_int64_int32_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB47_1
+ .p2align 4, 0x90
+.LBB47_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi], edx
+ mov rdx, qword ptr [rdi + 8]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 4], edx
+ mov rdx, qword ptr [rdi + 16]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 8], edx
+ mov rdx, qword ptr [rdi + 24]
+ mov edx, dword ptr [rcx + 4*rdx]
+ mov dword ptr [rsi + 12], edx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 16
+ cmp eax, 7
+ jg .LBB47_5
+.LBB47_1:
+ test edx, edx
+ jle .LBB47_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB47_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + 2*r8]
+ mov eax, dword ptr [rcx + 4*rax]
+ mov dword ptr [rsi + r8], eax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB47_3
+.LBB47_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end47:
+ .size transpose_int64_int32_sse4, .Lfunc_end47-transpose_int64_int32_sse4
+ # -- End function
+ .globl transpose_uint8_uint64_sse4 # -- Begin function transpose_uint8_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_uint64_sse4,@function
+transpose_uint8_uint64_sse4: # @transpose_uint8_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB48_1
+ .p2align 4, 0x90
+.LBB48_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB48_5
+.LBB48_1:
+ test edx, edx
+ jle .LBB48_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB48_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB48_3
+.LBB48_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end48:
+ .size transpose_uint8_uint64_sse4, .Lfunc_end48-transpose_uint8_uint64_sse4
+ # -- End function
+ .globl transpose_int8_uint64_sse4 # -- Begin function transpose_int8_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_uint64_sse4,@function
+transpose_int8_uint64_sse4: # @transpose_int8_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB49_1
+ .p2align 4, 0x90
+.LBB49_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB49_5
+.LBB49_1:
+ test edx, edx
+ jle .LBB49_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB49_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB49_3
+.LBB49_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end49:
+ .size transpose_int8_uint64_sse4, .Lfunc_end49-transpose_int8_uint64_sse4
+ # -- End function
+ .globl transpose_uint16_uint64_sse4 # -- Begin function transpose_uint16_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_uint64_sse4,@function
+transpose_uint16_uint64_sse4: # @transpose_uint16_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB50_1
+ .p2align 4, 0x90
+.LBB50_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB50_5
+.LBB50_1:
+ test edx, edx
+ jle .LBB50_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB50_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB50_3
+.LBB50_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end50:
+ .size transpose_uint16_uint64_sse4, .Lfunc_end50-transpose_uint16_uint64_sse4
+ # -- End function
+ .globl transpose_int16_uint64_sse4 # -- Begin function transpose_int16_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_uint64_sse4,@function
+transpose_int16_uint64_sse4: # @transpose_int16_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB51_1
+ .p2align 4, 0x90
+.LBB51_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB51_5
+.LBB51_1:
+ test edx, edx
+ jle .LBB51_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB51_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB51_3
+.LBB51_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end51:
+ .size transpose_int16_uint64_sse4, .Lfunc_end51-transpose_int16_uint64_sse4
+ # -- End function
+ .globl transpose_uint32_uint64_sse4 # -- Begin function transpose_uint32_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_uint64_sse4,@function
+transpose_uint32_uint64_sse4: # @transpose_uint32_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB52_1
+ .p2align 4, 0x90
+.LBB52_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov edx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov edx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov edx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB52_5
+.LBB52_1:
+ test edx, edx
+ jle .LBB52_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB52_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB52_3
+.LBB52_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end52:
+ .size transpose_uint32_uint64_sse4, .Lfunc_end52-transpose_uint32_uint64_sse4
+ # -- End function
+ .globl transpose_int32_uint64_sse4 # -- Begin function transpose_int32_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_uint64_sse4,@function
+transpose_int32_uint64_sse4: # @transpose_int32_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB53_1
+ .p2align 4, 0x90
+.LBB53_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsxd rdx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsxd rdx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsxd rdx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB53_5
+.LBB53_1:
+ test edx, edx
+ jle .LBB53_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB53_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB53_3
+.LBB53_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end53:
+ .size transpose_int32_uint64_sse4, .Lfunc_end53-transpose_int32_uint64_sse4
+ # -- End function
+ .globl transpose_uint64_uint64_sse4 # -- Begin function transpose_uint64_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_uint64_sse4,@function
+transpose_uint64_uint64_sse4: # @transpose_uint64_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB54_1
+ .p2align 4, 0x90
+.LBB54_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB54_5
+.LBB54_1:
+ test edx, edx
+ jle .LBB54_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB54_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB54_3
+.LBB54_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end54:
+ .size transpose_uint64_uint64_sse4, .Lfunc_end54-transpose_uint64_uint64_sse4
+ # -- End function
+ .globl transpose_int64_uint64_sse4 # -- Begin function transpose_int64_uint64_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_uint64_sse4,@function
+transpose_int64_uint64_sse4: # @transpose_int64_uint64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB55_1
+ .p2align 4, 0x90
+.LBB55_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB55_5
+.LBB55_1:
+ test edx, edx
+ jle .LBB55_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB55_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB55_3
+.LBB55_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end55:
+ .size transpose_int64_uint64_sse4, .Lfunc_end55-transpose_int64_uint64_sse4
+ # -- End function
+ .globl transpose_uint8_int64_sse4 # -- Begin function transpose_uint8_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint8_int64_sse4,@function
+transpose_uint8_int64_sse4: # @transpose_uint8_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB56_1
+ .p2align 4, 0x90
+.LBB56_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB56_5
+.LBB56_1:
+ test edx, edx
+ jle .LBB56_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB56_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB56_3
+.LBB56_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end56:
+ .size transpose_uint8_int64_sse4, .Lfunc_end56-transpose_uint8_int64_sse4
+ # -- End function
+ .globl transpose_int8_int64_sse4 # -- Begin function transpose_int8_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_int8_int64_sse4,@function
+transpose_int8_int64_sse4: # @transpose_int8_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB57_1
+ .p2align 4, 0x90
+.LBB57_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, byte ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, byte ptr [rdi + 1]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, byte ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, byte ptr [rdi + 3]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 4
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB57_5
+.LBB57_1:
+ test edx, edx
+ jle .LBB57_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB57_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, byte ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 8*r8], rax
+ add r8, 1
+ add edx, -1
+ cmp edx, 1
+ jg .LBB57_3
+.LBB57_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end57:
+ .size transpose_int8_int64_sse4, .Lfunc_end57-transpose_int8_int64_sse4
+ # -- End function
+ .globl transpose_uint16_int64_sse4 # -- Begin function transpose_uint16_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint16_int64_sse4,@function
+transpose_uint16_int64_sse4: # @transpose_uint16_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB58_1
+ .p2align 4, 0x90
+.LBB58_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movzx edx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movzx edx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movzx edx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movzx edx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB58_5
+.LBB58_1:
+ test edx, edx
+ jle .LBB58_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB58_3: # =>This Inner Loop Header: Depth=1
+ movzx eax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB58_3
+.LBB58_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end58:
+ .size transpose_uint16_int64_sse4, .Lfunc_end58-transpose_uint16_int64_sse4
+ # -- End function
+ .globl transpose_int16_int64_sse4 # -- Begin function transpose_int16_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_int16_int64_sse4,@function
+transpose_int16_int64_sse4: # @transpose_int16_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB59_1
+ .p2align 4, 0x90
+.LBB59_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsx rdx, word ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsx rdx, word ptr [rdi + 2]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsx rdx, word ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsx rdx, word ptr [rdi + 6]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 8
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB59_5
+.LBB59_1:
+ test edx, edx
+ jle .LBB59_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB59_3: # =>This Inner Loop Header: Depth=1
+ movsx rax, word ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 4*r8], rax
+ add r8, 2
+ add edx, -1
+ cmp edx, 1
+ jg .LBB59_3
+.LBB59_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end59:
+ .size transpose_int16_int64_sse4, .Lfunc_end59-transpose_int16_int64_sse4
+ # -- End function
+ .globl transpose_uint32_int64_sse4 # -- Begin function transpose_uint32_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint32_int64_sse4,@function
+transpose_uint32_int64_sse4: # @transpose_uint32_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB60_1
+ .p2align 4, 0x90
+.LBB60_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov edx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov edx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov edx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov edx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB60_5
+.LBB60_1:
+ test edx, edx
+ jle .LBB60_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB60_3: # =>This Inner Loop Header: Depth=1
+ mov eax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB60_3
+.LBB60_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end60:
+ .size transpose_uint32_int64_sse4, .Lfunc_end60-transpose_uint32_int64_sse4
+ # -- End function
+ .globl transpose_int32_int64_sse4 # -- Begin function transpose_int32_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_int32_int64_sse4,@function
+transpose_int32_int64_sse4: # @transpose_int32_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB61_1
+ .p2align 4, 0x90
+.LBB61_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ movsxd rdx, dword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ movsxd rdx, dword ptr [rdi + 4]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ movsxd rdx, dword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ movsxd rdx, dword ptr [rdi + 12]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 16
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB61_5
+.LBB61_1:
+ test edx, edx
+ jle .LBB61_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB61_3: # =>This Inner Loop Header: Depth=1
+ movsxd rax, dword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + 2*r8], rax
+ add r8, 4
+ add edx, -1
+ cmp edx, 1
+ jg .LBB61_3
+.LBB61_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end61:
+ .size transpose_int32_int64_sse4, .Lfunc_end61-transpose_int32_int64_sse4
+ # -- End function
+ .globl transpose_uint64_int64_sse4 # -- Begin function transpose_uint64_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_uint64_int64_sse4,@function
+transpose_uint64_int64_sse4: # @transpose_uint64_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB62_1
+ .p2align 4, 0x90
+.LBB62_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB62_5
+.LBB62_1:
+ test edx, edx
+ jle .LBB62_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB62_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB62_3
+.LBB62_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end62:
+ .size transpose_uint64_int64_sse4, .Lfunc_end62-transpose_uint64_int64_sse4
+ # -- End function
+ .globl transpose_int64_int64_sse4 # -- Begin function transpose_int64_int64_sse4
+ .p2align 4, 0x90
+ .type transpose_int64_int64_sse4,@function
+transpose_int64_int64_sse4: # @transpose_int64_int64_sse4
+# %bb.0:
+ push rbp
+ mov rbp, rsp
+ and rsp, -8
+ cmp edx, 4
+ jl .LBB63_1
+ .p2align 4, 0x90
+.LBB63_5: # =>This Inner Loop Header: Depth=1
+ mov eax, edx
+ mov rdx, qword ptr [rdi]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi], rdx
+ mov rdx, qword ptr [rdi + 8]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 8], rdx
+ mov rdx, qword ptr [rdi + 16]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 16], rdx
+ mov rdx, qword ptr [rdi + 24]
+ movsxd rdx, dword ptr [rcx + 4*rdx]
+ mov qword ptr [rsi + 24], rdx
+ lea edx, [rax - 4]
+ add rdi, 32
+ add rsi, 32
+ cmp eax, 7
+ jg .LBB63_5
+.LBB63_1:
+ test edx, edx
+ jle .LBB63_4
+# %bb.2:
+ add edx, 1
+ xor r8d, r8d
+ .p2align 4, 0x90
+.LBB63_3: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rdi + r8]
+ movsxd rax, dword ptr [rcx + 4*rax]
+ mov qword ptr [rsi + r8], rax
+ add r8, 8
+ add edx, -1
+ cmp edx, 1
+ jg .LBB63_3
+.LBB63_4:
+ mov rsp, rbp
+ pop rbp
+ ret
+.Lfunc_end63:
+ .size transpose_int64_int64_sse4, .Lfunc_end63-transpose_int64_int64_sse4
+ # -- End function
+ .ident "Ubuntu clang version 11.0.0-2~ubuntu20.04.1"
+ .section ".note.GNU-stack","",@progbits
+ .addrsig
diff --git a/go/internal/utils/transpose_ints.go b/go/internal/utils/transpose_ints.go
new file mode 100644
index 0000000000..1666df1296
--- /dev/null
+++ b/go/internal/utils/transpose_ints.go
@@ -0,0 +1,407 @@
+// Code generated by transpose_ints.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// when we upgrade to support go1.18, this can be massively simplified by using
+// Go Generics, but since we aren't supporting go1.18 yet, I didn't want to use
+// them here so we can maintain the backwards compatibility.
+
+func transposeInt8Int8(src []int8, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeInt8Uint8(src []int8, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeInt8Int16(src []int8, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeInt8Uint16(src []int8, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeInt8Int32(src []int8, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeInt8Uint32(src []int8, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeInt8Int64(src []int8, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeInt8Uint64(src []int8, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeUint8Int8(src []uint8, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeUint8Uint8(src []uint8, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeUint8Int16(src []uint8, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeUint8Uint16(src []uint8, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeUint8Int32(src []uint8, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeUint8Uint32(src []uint8, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeUint8Int64(src []uint8, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeUint8Uint64(src []uint8, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeInt16Int8(src []int16, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeInt16Uint8(src []int16, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeInt16Int16(src []int16, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeInt16Uint16(src []int16, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeInt16Int32(src []int16, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeInt16Uint32(src []int16, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeInt16Int64(src []int16, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeInt16Uint64(src []int16, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeUint16Int8(src []uint16, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeUint16Uint8(src []uint16, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeUint16Int16(src []uint16, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeUint16Uint16(src []uint16, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeUint16Int32(src []uint16, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeUint16Uint32(src []uint16, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeUint16Int64(src []uint16, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeUint16Uint64(src []uint16, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeInt32Int8(src []int32, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeInt32Uint8(src []int32, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeInt32Int16(src []int32, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeInt32Uint16(src []int32, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeInt32Int32(src []int32, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeInt32Uint32(src []int32, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeInt32Int64(src []int32, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeInt32Uint64(src []int32, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeUint32Int8(src []uint32, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeUint32Uint8(src []uint32, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeUint32Int16(src []uint32, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeUint32Uint16(src []uint32, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeUint32Int32(src []uint32, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeUint32Uint32(src []uint32, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeUint32Int64(src []uint32, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeUint32Uint64(src []uint32, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeInt64Int8(src []int64, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeInt64Uint8(src []int64, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeInt64Int16(src []int64, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeInt64Uint16(src []int64, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeInt64Int32(src []int64, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeInt64Uint32(src []int64, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeInt64Int64(src []int64, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeInt64Uint64(src []int64, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
+
+func transposeUint64Int8(src []uint64, dest []int8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int8(transposeMap[s])
+ }
+}
+
+func transposeUint64Uint8(src []uint64, dest []uint8, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint8(transposeMap[s])
+ }
+}
+
+func transposeUint64Int16(src []uint64, dest []int16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int16(transposeMap[s])
+ }
+}
+
+func transposeUint64Uint16(src []uint64, dest []uint16, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint16(transposeMap[s])
+ }
+}
+
+func transposeUint64Int32(src []uint64, dest []int32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int32(transposeMap[s])
+ }
+}
+
+func transposeUint64Uint32(src []uint64, dest []uint32, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint32(transposeMap[s])
+ }
+}
+
+func transposeUint64Int64(src []uint64, dest []int64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = int64(transposeMap[s])
+ }
+}
+
+func transposeUint64Uint64(src []uint64, dest []uint64, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = uint64(transposeMap[s])
+ }
+}
diff --git a/go/internal/utils/transpose_ints.go.tmpl b/go/internal/utils/transpose_ints.go.tmpl
new file mode 100644
index 0000000000..680ae1ee71
--- /dev/null
+++ b/go/internal/utils/transpose_ints.go.tmpl
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+{{ $typelist := .In }}
+{{range .In}}
+{{ $src := .Type }}
+{{ $srcName := .Name }}
+{{ range $typelist }}
+{{ $dest := .Type }}
+{{ $destName := .Name }}
+
+func transpose{{ $srcName }}{{ $destName }}(src []{{$src}}, dest []{{$dest}}, transposeMap []int32) {
+ for i, s := range src {
+ dest[i] = {{ $dest }}(transposeMap[s])
+ }
+}
+
+{{ end }}
+{{ end }}
diff --git a/go/internal/utils/transpose_ints.tmpldata b/go/internal/utils/transpose_ints.tmpldata
new file mode 100644
index 0000000000..72eaf300c4
--- /dev/null
+++ b/go/internal/utils/transpose_ints.tmpldata
@@ -0,0 +1,34 @@
+[
+ {
+ "Name": "Int8",
+ "Type": "int8"
+ },
+ {
+ "Name": "Uint8",
+ "Type": "uint8"
+ },
+ {
+ "Name": "Int16",
+ "Type": "int16"
+ },
+ {
+ "Name": "Uint16",
+ "Type": "uint16"
+ },
+ {
+ "Name": "Int32",
+ "Type": "int32"
+ },
+ {
+ "Name": "Uint32",
+ "Type": "uint32"
+ },
+ {
+ "Name": "Int64",
+ "Type": "int64"
+ },
+ {
+ "Name": "Uint64",
+ "Type": "uint64"
+ }
+]
diff --git a/go/internal/utils/transpose_ints_amd64.go b/go/internal/utils/transpose_ints_amd64.go
new file mode 100644
index 0000000000..0f7c252979
--- /dev/null
+++ b/go/internal/utils/transpose_ints_amd64.go
@@ -0,0 +1,326 @@
+// Code generated by transpose_ints_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noasm
+// +build !noasm
+
+package utils
+
+import (
+ "golang.org/x/sys/cpu"
+)
+
+var (
+ TransposeInt8Int8 func([]int8, []int8, []int32)
+ TransposeInt8Uint8 func([]int8, []uint8, []int32)
+ TransposeInt8Int16 func([]int8, []int16, []int32)
+ TransposeInt8Uint16 func([]int8, []uint16, []int32)
+ TransposeInt8Int32 func([]int8, []int32, []int32)
+ TransposeInt8Uint32 func([]int8, []uint32, []int32)
+ TransposeInt8Int64 func([]int8, []int64, []int32)
+ TransposeInt8Uint64 func([]int8, []uint64, []int32)
+
+ TransposeUint8Int8 func([]uint8, []int8, []int32)
+ TransposeUint8Uint8 func([]uint8, []uint8, []int32)
+ TransposeUint8Int16 func([]uint8, []int16, []int32)
+ TransposeUint8Uint16 func([]uint8, []uint16, []int32)
+ TransposeUint8Int32 func([]uint8, []int32, []int32)
+ TransposeUint8Uint32 func([]uint8, []uint32, []int32)
+ TransposeUint8Int64 func([]uint8, []int64, []int32)
+ TransposeUint8Uint64 func([]uint8, []uint64, []int32)
+
+ TransposeInt16Int8 func([]int16, []int8, []int32)
+ TransposeInt16Uint8 func([]int16, []uint8, []int32)
+ TransposeInt16Int16 func([]int16, []int16, []int32)
+ TransposeInt16Uint16 func([]int16, []uint16, []int32)
+ TransposeInt16Int32 func([]int16, []int32, []int32)
+ TransposeInt16Uint32 func([]int16, []uint32, []int32)
+ TransposeInt16Int64 func([]int16, []int64, []int32)
+ TransposeInt16Uint64 func([]int16, []uint64, []int32)
+
+ TransposeUint16Int8 func([]uint16, []int8, []int32)
+ TransposeUint16Uint8 func([]uint16, []uint8, []int32)
+ TransposeUint16Int16 func([]uint16, []int16, []int32)
+ TransposeUint16Uint16 func([]uint16, []uint16, []int32)
+ TransposeUint16Int32 func([]uint16, []int32, []int32)
+ TransposeUint16Uint32 func([]uint16, []uint32, []int32)
+ TransposeUint16Int64 func([]uint16, []int64, []int32)
+ TransposeUint16Uint64 func([]uint16, []uint64, []int32)
+
+ TransposeInt32Int8 func([]int32, []int8, []int32)
+ TransposeInt32Uint8 func([]int32, []uint8, []int32)
+ TransposeInt32Int16 func([]int32, []int16, []int32)
+ TransposeInt32Uint16 func([]int32, []uint16, []int32)
+ TransposeInt32Int32 func([]int32, []int32, []int32)
+ TransposeInt32Uint32 func([]int32, []uint32, []int32)
+ TransposeInt32Int64 func([]int32, []int64, []int32)
+ TransposeInt32Uint64 func([]int32, []uint64, []int32)
+
+ TransposeUint32Int8 func([]uint32, []int8, []int32)
+ TransposeUint32Uint8 func([]uint32, []uint8, []int32)
+ TransposeUint32Int16 func([]uint32, []int16, []int32)
+ TransposeUint32Uint16 func([]uint32, []uint16, []int32)
+ TransposeUint32Int32 func([]uint32, []int32, []int32)
+ TransposeUint32Uint32 func([]uint32, []uint32, []int32)
+ TransposeUint32Int64 func([]uint32, []int64, []int32)
+ TransposeUint32Uint64 func([]uint32, []uint64, []int32)
+
+ TransposeInt64Int8 func([]int64, []int8, []int32)
+ TransposeInt64Uint8 func([]int64, []uint8, []int32)
+ TransposeInt64Int16 func([]int64, []int16, []int32)
+ TransposeInt64Uint16 func([]int64, []uint16, []int32)
+ TransposeInt64Int32 func([]int64, []int32, []int32)
+ TransposeInt64Uint32 func([]int64, []uint32, []int32)
+ TransposeInt64Int64 func([]int64, []int64, []int32)
+ TransposeInt64Uint64 func([]int64, []uint64, []int32)
+
+ TransposeUint64Int8 func([]uint64, []int8, []int32)
+ TransposeUint64Uint8 func([]uint64, []uint8, []int32)
+ TransposeUint64Int16 func([]uint64, []int16, []int32)
+ TransposeUint64Uint16 func([]uint64, []uint16, []int32)
+ TransposeUint64Int32 func([]uint64, []int32, []int32)
+ TransposeUint64Uint32 func([]uint64, []uint32, []int32)
+ TransposeUint64Int64 func([]uint64, []int64, []int32)
+ TransposeUint64Uint64 func([]uint64, []uint64, []int32)
+)
+
+func init() {
+ if cpu.X86.HasAVX2 {
+
+ TransposeInt8Int8 = transposeInt8Int8avx2
+ TransposeInt8Uint8 = transposeInt8Uint8avx2
+ TransposeInt8Int16 = transposeInt8Int16avx2
+ TransposeInt8Uint16 = transposeInt8Uint16avx2
+ TransposeInt8Int32 = transposeInt8Int32avx2
+ TransposeInt8Uint32 = transposeInt8Uint32avx2
+ TransposeInt8Int64 = transposeInt8Int64avx2
+ TransposeInt8Uint64 = transposeInt8Uint64avx2
+
+ TransposeUint8Int8 = transposeUint8Int8avx2
+ TransposeUint8Uint8 = transposeUint8Uint8avx2
+ TransposeUint8Int16 = transposeUint8Int16avx2
+ TransposeUint8Uint16 = transposeUint8Uint16avx2
+ TransposeUint8Int32 = transposeUint8Int32avx2
+ TransposeUint8Uint32 = transposeUint8Uint32avx2
+ TransposeUint8Int64 = transposeUint8Int64avx2
+ TransposeUint8Uint64 = transposeUint8Uint64avx2
+
+ TransposeInt16Int8 = transposeInt16Int8avx2
+ TransposeInt16Uint8 = transposeInt16Uint8avx2
+ TransposeInt16Int16 = transposeInt16Int16avx2
+ TransposeInt16Uint16 = transposeInt16Uint16avx2
+ TransposeInt16Int32 = transposeInt16Int32avx2
+ TransposeInt16Uint32 = transposeInt16Uint32avx2
+ TransposeInt16Int64 = transposeInt16Int64avx2
+ TransposeInt16Uint64 = transposeInt16Uint64avx2
+
+ TransposeUint16Int8 = transposeUint16Int8avx2
+ TransposeUint16Uint8 = transposeUint16Uint8avx2
+ TransposeUint16Int16 = transposeUint16Int16avx2
+ TransposeUint16Uint16 = transposeUint16Uint16avx2
+ TransposeUint16Int32 = transposeUint16Int32avx2
+ TransposeUint16Uint32 = transposeUint16Uint32avx2
+ TransposeUint16Int64 = transposeUint16Int64avx2
+ TransposeUint16Uint64 = transposeUint16Uint64avx2
+
+ TransposeInt32Int8 = transposeInt32Int8avx2
+ TransposeInt32Uint8 = transposeInt32Uint8avx2
+ TransposeInt32Int16 = transposeInt32Int16avx2
+ TransposeInt32Uint16 = transposeInt32Uint16avx2
+ TransposeInt32Int32 = transposeInt32Int32avx2
+ TransposeInt32Uint32 = transposeInt32Uint32avx2
+ TransposeInt32Int64 = transposeInt32Int64avx2
+ TransposeInt32Uint64 = transposeInt32Uint64avx2
+
+ TransposeUint32Int8 = transposeUint32Int8avx2
+ TransposeUint32Uint8 = transposeUint32Uint8avx2
+ TransposeUint32Int16 = transposeUint32Int16avx2
+ TransposeUint32Uint16 = transposeUint32Uint16avx2
+ TransposeUint32Int32 = transposeUint32Int32avx2
+ TransposeUint32Uint32 = transposeUint32Uint32avx2
+ TransposeUint32Int64 = transposeUint32Int64avx2
+ TransposeUint32Uint64 = transposeUint32Uint64avx2
+
+ TransposeInt64Int8 = transposeInt64Int8avx2
+ TransposeInt64Uint8 = transposeInt64Uint8avx2
+ TransposeInt64Int16 = transposeInt64Int16avx2
+ TransposeInt64Uint16 = transposeInt64Uint16avx2
+ TransposeInt64Int32 = transposeInt64Int32avx2
+ TransposeInt64Uint32 = transposeInt64Uint32avx2
+ TransposeInt64Int64 = transposeInt64Int64avx2
+ TransposeInt64Uint64 = transposeInt64Uint64avx2
+
+ TransposeUint64Int8 = transposeUint64Int8avx2
+ TransposeUint64Uint8 = transposeUint64Uint8avx2
+ TransposeUint64Int16 = transposeUint64Int16avx2
+ TransposeUint64Uint16 = transposeUint64Uint16avx2
+ TransposeUint64Int32 = transposeUint64Int32avx2
+ TransposeUint64Uint32 = transposeUint64Uint32avx2
+ TransposeUint64Int64 = transposeUint64Int64avx2
+ TransposeUint64Uint64 = transposeUint64Uint64avx2
+
+ } else if cpu.X86.HasSSE42 {
+
+ TransposeInt8Int8 = transposeInt8Int8sse4
+ TransposeInt8Uint8 = transposeInt8Uint8sse4
+ TransposeInt8Int16 = transposeInt8Int16sse4
+ TransposeInt8Uint16 = transposeInt8Uint16sse4
+ TransposeInt8Int32 = transposeInt8Int32sse4
+ TransposeInt8Uint32 = transposeInt8Uint32sse4
+ TransposeInt8Int64 = transposeInt8Int64sse4
+ TransposeInt8Uint64 = transposeInt8Uint64sse4
+
+ TransposeUint8Int8 = transposeUint8Int8sse4
+ TransposeUint8Uint8 = transposeUint8Uint8sse4
+ TransposeUint8Int16 = transposeUint8Int16sse4
+ TransposeUint8Uint16 = transposeUint8Uint16sse4
+ TransposeUint8Int32 = transposeUint8Int32sse4
+ TransposeUint8Uint32 = transposeUint8Uint32sse4
+ TransposeUint8Int64 = transposeUint8Int64sse4
+ TransposeUint8Uint64 = transposeUint8Uint64sse4
+
+ TransposeInt16Int8 = transposeInt16Int8sse4
+ TransposeInt16Uint8 = transposeInt16Uint8sse4
+ TransposeInt16Int16 = transposeInt16Int16sse4
+ TransposeInt16Uint16 = transposeInt16Uint16sse4
+ TransposeInt16Int32 = transposeInt16Int32sse4
+ TransposeInt16Uint32 = transposeInt16Uint32sse4
+ TransposeInt16Int64 = transposeInt16Int64sse4
+ TransposeInt16Uint64 = transposeInt16Uint64sse4
+
+ TransposeUint16Int8 = transposeUint16Int8sse4
+ TransposeUint16Uint8 = transposeUint16Uint8sse4
+ TransposeUint16Int16 = transposeUint16Int16sse4
+ TransposeUint16Uint16 = transposeUint16Uint16sse4
+ TransposeUint16Int32 = transposeUint16Int32sse4
+ TransposeUint16Uint32 = transposeUint16Uint32sse4
+ TransposeUint16Int64 = transposeUint16Int64sse4
+ TransposeUint16Uint64 = transposeUint16Uint64sse4
+
+ TransposeInt32Int8 = transposeInt32Int8sse4
+ TransposeInt32Uint8 = transposeInt32Uint8sse4
+ TransposeInt32Int16 = transposeInt32Int16sse4
+ TransposeInt32Uint16 = transposeInt32Uint16sse4
+ TransposeInt32Int32 = transposeInt32Int32sse4
+ TransposeInt32Uint32 = transposeInt32Uint32sse4
+ TransposeInt32Int64 = transposeInt32Int64sse4
+ TransposeInt32Uint64 = transposeInt32Uint64sse4
+
+ TransposeUint32Int8 = transposeUint32Int8sse4
+ TransposeUint32Uint8 = transposeUint32Uint8sse4
+ TransposeUint32Int16 = transposeUint32Int16sse4
+ TransposeUint32Uint16 = transposeUint32Uint16sse4
+ TransposeUint32Int32 = transposeUint32Int32sse4
+ TransposeUint32Uint32 = transposeUint32Uint32sse4
+ TransposeUint32Int64 = transposeUint32Int64sse4
+ TransposeUint32Uint64 = transposeUint32Uint64sse4
+
+ TransposeInt64Int8 = transposeInt64Int8sse4
+ TransposeInt64Uint8 = transposeInt64Uint8sse4
+ TransposeInt64Int16 = transposeInt64Int16sse4
+ TransposeInt64Uint16 = transposeInt64Uint16sse4
+ TransposeInt64Int32 = transposeInt64Int32sse4
+ TransposeInt64Uint32 = transposeInt64Uint32sse4
+ TransposeInt64Int64 = transposeInt64Int64sse4
+ TransposeInt64Uint64 = transposeInt64Uint64sse4
+
+ TransposeUint64Int8 = transposeUint64Int8sse4
+ TransposeUint64Uint8 = transposeUint64Uint8sse4
+ TransposeUint64Int16 = transposeUint64Int16sse4
+ TransposeUint64Uint16 = transposeUint64Uint16sse4
+ TransposeUint64Int32 = transposeUint64Int32sse4
+ TransposeUint64Uint32 = transposeUint64Uint32sse4
+ TransposeUint64Int64 = transposeUint64Int64sse4
+ TransposeUint64Uint64 = transposeUint64Uint64sse4
+
+ } else {
+
+ TransposeInt8Int8 = transposeInt8Int8
+ TransposeInt8Uint8 = transposeInt8Uint8
+ TransposeInt8Int16 = transposeInt8Int16
+ TransposeInt8Uint16 = transposeInt8Uint16
+ TransposeInt8Int32 = transposeInt8Int32
+ TransposeInt8Uint32 = transposeInt8Uint32
+ TransposeInt8Int64 = transposeInt8Int64
+ TransposeInt8Uint64 = transposeInt8Uint64
+
+ TransposeUint8Int8 = transposeUint8Int8
+ TransposeUint8Uint8 = transposeUint8Uint8
+ TransposeUint8Int16 = transposeUint8Int16
+ TransposeUint8Uint16 = transposeUint8Uint16
+ TransposeUint8Int32 = transposeUint8Int32
+ TransposeUint8Uint32 = transposeUint8Uint32
+ TransposeUint8Int64 = transposeUint8Int64
+ TransposeUint8Uint64 = transposeUint8Uint64
+
+ TransposeInt16Int8 = transposeInt16Int8
+ TransposeInt16Uint8 = transposeInt16Uint8
+ TransposeInt16Int16 = transposeInt16Int16
+ TransposeInt16Uint16 = transposeInt16Uint16
+ TransposeInt16Int32 = transposeInt16Int32
+ TransposeInt16Uint32 = transposeInt16Uint32
+ TransposeInt16Int64 = transposeInt16Int64
+ TransposeInt16Uint64 = transposeInt16Uint64
+
+ TransposeUint16Int8 = transposeUint16Int8
+ TransposeUint16Uint8 = transposeUint16Uint8
+ TransposeUint16Int16 = transposeUint16Int16
+ TransposeUint16Uint16 = transposeUint16Uint16
+ TransposeUint16Int32 = transposeUint16Int32
+ TransposeUint16Uint32 = transposeUint16Uint32
+ TransposeUint16Int64 = transposeUint16Int64
+ TransposeUint16Uint64 = transposeUint16Uint64
+
+ TransposeInt32Int8 = transposeInt32Int8
+ TransposeInt32Uint8 = transposeInt32Uint8
+ TransposeInt32Int16 = transposeInt32Int16
+ TransposeInt32Uint16 = transposeInt32Uint16
+ TransposeInt32Int32 = transposeInt32Int32
+ TransposeInt32Uint32 = transposeInt32Uint32
+ TransposeInt32Int64 = transposeInt32Int64
+ TransposeInt32Uint64 = transposeInt32Uint64
+
+ TransposeUint32Int8 = transposeUint32Int8
+ TransposeUint32Uint8 = transposeUint32Uint8
+ TransposeUint32Int16 = transposeUint32Int16
+ TransposeUint32Uint16 = transposeUint32Uint16
+ TransposeUint32Int32 = transposeUint32Int32
+ TransposeUint32Uint32 = transposeUint32Uint32
+ TransposeUint32Int64 = transposeUint32Int64
+ TransposeUint32Uint64 = transposeUint32Uint64
+
+ TransposeInt64Int8 = transposeInt64Int8
+ TransposeInt64Uint8 = transposeInt64Uint8
+ TransposeInt64Int16 = transposeInt64Int16
+ TransposeInt64Uint16 = transposeInt64Uint16
+ TransposeInt64Int32 = transposeInt64Int32
+ TransposeInt64Uint32 = transposeInt64Uint32
+ TransposeInt64Int64 = transposeInt64Int64
+ TransposeInt64Uint64 = transposeInt64Uint64
+
+ TransposeUint64Int8 = transposeUint64Int8
+ TransposeUint64Uint8 = transposeUint64Uint8
+ TransposeUint64Int16 = transposeUint64Int16
+ TransposeUint64Uint16 = transposeUint64Uint16
+ TransposeUint64Int32 = transposeUint64Int32
+ TransposeUint64Uint32 = transposeUint64Uint32
+ TransposeUint64Int64 = transposeUint64Int64
+ TransposeUint64Uint64 = transposeUint64Uint64
+
+ }
+}
diff --git a/go/internal/utils/transpose_ints_amd64.go.tmpl b/go/internal/utils/transpose_ints_amd64.go.tmpl
new file mode 100644
index 0000000000..eac0208e5e
--- /dev/null
+++ b/go/internal/utils/transpose_ints_amd64.go.tmpl
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noasm
+// +build !noasm
+
+package utils
+
+import (
+ "golang.org/x/sys/cpu"
+)
+
+var (
+{{ $typelist := .In }}
+{{range .In}}
+{{ $src := .Type -}}
+{{ $srcName := .Name -}}
+{{ range $typelist -}}
+{{ $dest := .Type -}}
+{{ $destName := .Name -}}
+ Transpose{{$srcName}}{{$destName}} func([]{{$src}}, []{{$dest}}, []int32)
+{{end}}
+{{end}}
+)
+
+
+func init() {
+ if cpu.X86.HasAVX2 {
+{{ $typelist := .In }}
+{{range .In}}
+{{ $src := .Type -}}
+{{ $srcName := .Name -}}
+{{ range $typelist -}}
+{{ $dest := .Type -}}
+{{ $destName := .Name -}}
+ Transpose{{$srcName}}{{$destName}} = transpose{{ $srcName }}{{ $destName }}avx2
+{{end}}
+{{end}}
+ } else if cpu.X86.HasSSE42 {
+{{ $typelist := .In }}
+{{range .In}}
+{{ $src := .Type -}}
+{{ $srcName := .Name -}}
+{{ range $typelist -}}
+{{ $dest := .Type -}}
+{{ $destName := .Name -}}
+ Transpose{{$srcName}}{{$destName}} = transpose{{ $srcName }}{{ $destName }}sse4
+{{end}}
+{{end}}
+ } else {
+{{ $typelist := .In }}
+{{range .In}}
+{{ $src := .Type -}}
+{{ $srcName := .Name -}}
+{{ range $typelist -}}
+{{ $dest := .Type -}}
+{{ $destName := .Name -}}
+ Transpose{{$srcName}}{{$destName}} = transpose{{ $srcName }}{{ $destName }}
+{{end}}
+{{end}}
+ }
+}
diff --git a/go/internal/utils/transpose_ints_arm64.go b/go/internal/utils/transpose_ints_arm64.go
new file mode 100644
index 0000000000..2650cebe75
--- /dev/null
+++ b/go/internal/utils/transpose_ints_arm64.go
@@ -0,0 +1,97 @@
+// Code generated by transpose_ints_s390x.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noasm
+// +build !noasm
+
+package utils
+
+// if building with the 'noasm' tag, then point to the pure go implementations
+var (
+ TransposeInt8Int8 = transposeInt8Int8
+ TransposeInt8Uint8 = transposeInt8Uint8
+ TransposeInt8Int16 = transposeInt8Int16
+ TransposeInt8Uint16 = transposeInt8Uint16
+ TransposeInt8Int32 = transposeInt8Int32
+ TransposeInt8Uint32 = transposeInt8Uint32
+ TransposeInt8Int64 = transposeInt8Int64
+ TransposeInt8Uint64 = transposeInt8Uint64
+
+ TransposeUint8Int8 = transposeUint8Int8
+ TransposeUint8Uint8 = transposeUint8Uint8
+ TransposeUint8Int16 = transposeUint8Int16
+ TransposeUint8Uint16 = transposeUint8Uint16
+ TransposeUint8Int32 = transposeUint8Int32
+ TransposeUint8Uint32 = transposeUint8Uint32
+ TransposeUint8Int64 = transposeUint8Int64
+ TransposeUint8Uint64 = transposeUint8Uint64
+
+ TransposeInt16Int8 = transposeInt16Int8
+ TransposeInt16Uint8 = transposeInt16Uint8
+ TransposeInt16Int16 = transposeInt16Int16
+ TransposeInt16Uint16 = transposeInt16Uint16
+ TransposeInt16Int32 = transposeInt16Int32
+ TransposeInt16Uint32 = transposeInt16Uint32
+ TransposeInt16Int64 = transposeInt16Int64
+ TransposeInt16Uint64 = transposeInt16Uint64
+
+ TransposeUint16Int8 = transposeUint16Int8
+ TransposeUint16Uint8 = transposeUint16Uint8
+ TransposeUint16Int16 = transposeUint16Int16
+ TransposeUint16Uint16 = transposeUint16Uint16
+ TransposeUint16Int32 = transposeUint16Int32
+ TransposeUint16Uint32 = transposeUint16Uint32
+ TransposeUint16Int64 = transposeUint16Int64
+ TransposeUint16Uint64 = transposeUint16Uint64
+
+ TransposeInt32Int8 = transposeInt32Int8
+ TransposeInt32Uint8 = transposeInt32Uint8
+ TransposeInt32Int16 = transposeInt32Int16
+ TransposeInt32Uint16 = transposeInt32Uint16
+ TransposeInt32Int32 = transposeInt32Int32
+ TransposeInt32Uint32 = transposeInt32Uint32
+ TransposeInt32Int64 = transposeInt32Int64
+ TransposeInt32Uint64 = transposeInt32Uint64
+
+ TransposeUint32Int8 = transposeUint32Int8
+ TransposeUint32Uint8 = transposeUint32Uint8
+ TransposeUint32Int16 = transposeUint32Int16
+ TransposeUint32Uint16 = transposeUint32Uint16
+ TransposeUint32Int32 = transposeUint32Int32
+ TransposeUint32Uint32 = transposeUint32Uint32
+ TransposeUint32Int64 = transposeUint32Int64
+ TransposeUint32Uint64 = transposeUint32Uint64
+
+ TransposeInt64Int8 = transposeInt64Int8
+ TransposeInt64Uint8 = transposeInt64Uint8
+ TransposeInt64Int16 = transposeInt64Int16
+ TransposeInt64Uint16 = transposeInt64Uint16
+ TransposeInt64Int32 = transposeInt64Int32
+ TransposeInt64Uint32 = transposeInt64Uint32
+ TransposeInt64Int64 = transposeInt64Int64
+ TransposeInt64Uint64 = transposeInt64Uint64
+
+ TransposeUint64Int8 = transposeUint64Int8
+ TransposeUint64Uint8 = transposeUint64Uint8
+ TransposeUint64Int16 = transposeUint64Int16
+ TransposeUint64Uint16 = transposeUint64Uint16
+ TransposeUint64Int32 = transposeUint64Int32
+ TransposeUint64Uint32 = transposeUint64Uint32
+ TransposeUint64Int64 = transposeUint64Int64
+ TransposeUint64Uint64 = transposeUint64Uint64
+)
diff --git a/go/internal/utils/transpose_ints_avx2_amd64.go b/go/internal/utils/transpose_ints_avx2_amd64.go
new file mode 100644
index 0000000000..f2c73354a9
--- /dev/null
+++ b/go/internal/utils/transpose_ints_avx2_amd64.go
@@ -0,0 +1,474 @@
+// Code generated by transpose_ints_simd.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noasm
+// +build !noasm
+
+package utils
+
+import (
+ "unsafe"
+)
+
+//go:noescape
+func _transpose_int8_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Int8avx2(src []int8, dest []int8, transposeMap []int32) {
+ _transpose_int8_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Uint8avx2(src []int8, dest []uint8, transposeMap []int32) {
+ _transpose_int8_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Int16avx2(src []int8, dest []int16, transposeMap []int32) {
+ _transpose_int8_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Uint16avx2(src []int8, dest []uint16, transposeMap []int32) {
+ _transpose_int8_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Int32avx2(src []int8, dest []int32, transposeMap []int32) {
+ _transpose_int8_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Uint32avx2(src []int8, dest []uint32, transposeMap []int32) {
+ _transpose_int8_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Int64avx2(src []int8, dest []int64, transposeMap []int32) {
+ _transpose_int8_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int8_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt8Uint64avx2(src []int8, dest []uint64, transposeMap []int32) {
+ _transpose_int8_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Int8avx2(src []uint8, dest []int8, transposeMap []int32) {
+ _transpose_uint8_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Uint8avx2(src []uint8, dest []uint8, transposeMap []int32) {
+ _transpose_uint8_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Int16avx2(src []uint8, dest []int16, transposeMap []int32) {
+ _transpose_uint8_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Uint16avx2(src []uint8, dest []uint16, transposeMap []int32) {
+ _transpose_uint8_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Int32avx2(src []uint8, dest []int32, transposeMap []int32) {
+ _transpose_uint8_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Uint32avx2(src []uint8, dest []uint32, transposeMap []int32) {
+ _transpose_uint8_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Int64avx2(src []uint8, dest []int64, transposeMap []int32) {
+ _transpose_uint8_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint8_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint8Uint64avx2(src []uint8, dest []uint64, transposeMap []int32) {
+ _transpose_uint8_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Int8avx2(src []int16, dest []int8, transposeMap []int32) {
+ _transpose_int16_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Uint8avx2(src []int16, dest []uint8, transposeMap []int32) {
+ _transpose_int16_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Int16avx2(src []int16, dest []int16, transposeMap []int32) {
+ _transpose_int16_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Uint16avx2(src []int16, dest []uint16, transposeMap []int32) {
+ _transpose_int16_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Int32avx2(src []int16, dest []int32, transposeMap []int32) {
+ _transpose_int16_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Uint32avx2(src []int16, dest []uint32, transposeMap []int32) {
+ _transpose_int16_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Int64avx2(src []int16, dest []int64, transposeMap []int32) {
+ _transpose_int16_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int16_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt16Uint64avx2(src []int16, dest []uint64, transposeMap []int32) {
+ _transpose_int16_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Int8avx2(src []uint16, dest []int8, transposeMap []int32) {
+ _transpose_uint16_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Uint8avx2(src []uint16, dest []uint8, transposeMap []int32) {
+ _transpose_uint16_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Int16avx2(src []uint16, dest []int16, transposeMap []int32) {
+ _transpose_uint16_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Uint16avx2(src []uint16, dest []uint16, transposeMap []int32) {
+ _transpose_uint16_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Int32avx2(src []uint16, dest []int32, transposeMap []int32) {
+ _transpose_uint16_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Uint32avx2(src []uint16, dest []uint32, transposeMap []int32) {
+ _transpose_uint16_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Int64avx2(src []uint16, dest []int64, transposeMap []int32) {
+ _transpose_uint16_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint16_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint16Uint64avx2(src []uint16, dest []uint64, transposeMap []int32) {
+ _transpose_uint16_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Int8avx2(src []int32, dest []int8, transposeMap []int32) {
+ _transpose_int32_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Uint8avx2(src []int32, dest []uint8, transposeMap []int32) {
+ _transpose_int32_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Int16avx2(src []int32, dest []int16, transposeMap []int32) {
+ _transpose_int32_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Uint16avx2(src []int32, dest []uint16, transposeMap []int32) {
+ _transpose_int32_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Int32avx2(src []int32, dest []int32, transposeMap []int32) {
+ _transpose_int32_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Uint32avx2(src []int32, dest []uint32, transposeMap []int32) {
+ _transpose_int32_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Int64avx2(src []int32, dest []int64, transposeMap []int32) {
+ _transpose_int32_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int32_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt32Uint64avx2(src []int32, dest []uint64, transposeMap []int32) {
+ _transpose_int32_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Int8avx2(src []uint32, dest []int8, transposeMap []int32) {
+ _transpose_uint32_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Uint8avx2(src []uint32, dest []uint8, transposeMap []int32) {
+ _transpose_uint32_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Int16avx2(src []uint32, dest []int16, transposeMap []int32) {
+ _transpose_uint32_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Uint16avx2(src []uint32, dest []uint16, transposeMap []int32) {
+ _transpose_uint32_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Int32avx2(src []uint32, dest []int32, transposeMap []int32) {
+ _transpose_uint32_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Uint32avx2(src []uint32, dest []uint32, transposeMap []int32) {
+ _transpose_uint32_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Int64avx2(src []uint32, dest []int64, transposeMap []int32) {
+ _transpose_uint32_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint32_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint32Uint64avx2(src []uint32, dest []uint64, transposeMap []int32) {
+ _transpose_uint32_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Int8avx2(src []int64, dest []int8, transposeMap []int32) {
+ _transpose_int64_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Uint8avx2(src []int64, dest []uint8, transposeMap []int32) {
+ _transpose_int64_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Int16avx2(src []int64, dest []int16, transposeMap []int32) {
+ _transpose_int64_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Uint16avx2(src []int64, dest []uint16, transposeMap []int32) {
+ _transpose_int64_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Int32avx2(src []int64, dest []int32, transposeMap []int32) {
+ _transpose_int64_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Uint32avx2(src []int64, dest []uint32, transposeMap []int32) {
+ _transpose_int64_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Int64avx2(src []int64, dest []int64, transposeMap []int32) {
+ _transpose_int64_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_int64_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeInt64Uint64avx2(src []int64, dest []uint64, transposeMap []int32) {
+ _transpose_int64_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_int8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Int8avx2(src []uint64, dest []int8, transposeMap []int32) {
+ _transpose_uint64_int8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_uint8_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Uint8avx2(src []uint64, dest []uint8, transposeMap []int32) {
+ _transpose_uint64_uint8_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_int16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Int16avx2(src []uint64, dest []int16, transposeMap []int32) {
+ _transpose_uint64_int16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_uint16_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Uint16avx2(src []uint64, dest []uint16, transposeMap []int32) {
+ _transpose_uint64_uint16_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_int32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Int32avx2(src []uint64, dest []int32, transposeMap []int32) {
+ _transpose_uint64_int32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_uint32_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Uint32avx2(src []uint64, dest []uint32, transposeMap []int32) {
+ _transpose_uint64_uint32_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_int64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Int64avx2(src []uint64, dest []int64, transposeMap []int32) {
+ _transpose_uint64_int64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
+
+//go:noescape
+func _transpose_uint64_uint64_avx2(src, dest unsafe.Pointer, length int, transposeMap unsafe.Pointer)
+
+func transposeUint64Uint64avx2(src []uint64, dest []uint64, transposeMap []int32) {
+ _transpose_uint64_uint64_avx2(unsafe.Pointer(&src[0]), unsafe.Pointer(&dest[0]), len(dest), unsafe.Pointer(&transposeMap[0]))
+}
diff --git a/go/internal/utils/transpose_ints_avx2_amd64.s b/go/internal/utils/transpose_ints_avx2_amd64.s
new file mode 100644
index 0000000000..fbcc101eb2
--- /dev/null
+++ b/go/internal/utils/transpose_ints_avx2_amd64.s
@@ -0,0 +1,3074 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_transpose_uint8_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB0_1
+
+LBB0_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0xb60f; BYTE $0x17 // movzx edx, byte [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x0157b60f // movzx edx, byte [rdi + 1]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x0257b60f // movzx edx, byte [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x0357b60f // movzx edx, byte [rdi + 3]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x04c78348 // add rdi, 4
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB0_5
+
+LBB0_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB0_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB0_3:
+ LONG $0x04b60f42; BYTE $0x07 // movzx eax, byte [rdi + r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB0_3
+
+LBB0_4:
+ RET
+
+TEXT ·_transpose_int8_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB1_1
+
+LBB1_5:
+ WORD $0xd089 // mov eax, edx
+ LONG $0x17be0f48 // movsx rdx, byte [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x57be0f48; BYTE $0x01 // movsx rdx, byte [rdi + 1]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x57be0f48; BYTE $0x02 // movsx rdx, byte [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x57be0f48; BYTE $0x03 // movsx rdx, byte [rdi + 3]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x04c78348 // add rdi, 4
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB1_5
+
+LBB1_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB1_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB1_3:
+ LONG $0x04be0f4a; BYTE $0x07 // movsx rax, byte [rdi + r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB1_3
+
+LBB1_4:
+ RET
+
+TEXT ·_transpose_uint16_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB2_1
+
+LBB2_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0xb70f; BYTE $0x17 // movzx edx, word [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x0257b70f // movzx edx, word [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x0457b70f // movzx edx, word [rdi + 4]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x0657b70f // movzx edx, word [rdi + 6]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x08c78348 // add rdi, 8
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB2_5
+
+LBB2_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB2_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB2_3:
+ LONG $0x04b70f42; BYTE $0x47 // movzx eax, word [rdi + 2*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB2_3
+
+LBB2_4:
+ RET
+
+TEXT ·_transpose_int16_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB3_1
+
+LBB3_5:
+ WORD $0xd089 // mov eax, edx
+ LONG $0x17bf0f48 // movsx rdx, word [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x57bf0f48; BYTE $0x02 // movsx rdx, word [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x57bf0f48; BYTE $0x04 // movsx rdx, word [rdi + 4]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x57bf0f48; BYTE $0x06 // movsx rdx, word [rdi + 6]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x08c78348 // add rdi, 8
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB3_5
+
+LBB3_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB3_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB3_3:
+ LONG $0x04bf0f4a; BYTE $0x47 // movsx rax, word [rdi + 2*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB3_3
+
+LBB3_4:
+ RET
+
+TEXT ·_transpose_uint32_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB4_1
+
+LBB4_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0x178b // mov edx, dword [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ WORD $0x578b; BYTE $0x04 // mov edx, dword [rdi + 4]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ WORD $0x578b; BYTE $0x08 // mov edx, dword [rdi + 8]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ WORD $0x578b; BYTE $0x0c // mov edx, dword [rdi + 12]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x10c78348 // add rdi, 16
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB4_5
+
+LBB4_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB4_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB4_3:
+ LONG $0x87048b42 // mov eax, dword [rdi + 4*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB4_3
+
+LBB4_4:
+ RET
+
+TEXT ·_transpose_int32_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB5_1
+
+LBB5_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0x6348; BYTE $0x17 // movsxd rdx, dword [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x04576348 // movsxd rdx, dword [rdi + 4]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x08576348 // movsxd rdx, dword [rdi + 8]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x0c576348 // movsxd rdx, dword [rdi + 12]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x10c78348 // add rdi, 16
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB5_5
+
+LBB5_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB5_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB5_3:
+ LONG $0x8704634a // movsxd rax, dword [rdi + 4*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB5_3
+
+LBB5_4:
+ RET
+
+TEXT ·_transpose_uint64_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB6_1
+
+LBB6_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0x8b48; BYTE $0x17 // mov rdx, qword [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x08578b48 // mov rdx, qword [rdi + 8]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x10578b48 // mov rdx, qword [rdi + 16]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x18578b48 // mov rdx, qword [rdi + 24]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x20c78348 // add rdi, 32
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB6_5
+
+LBB6_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB6_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB6_3:
+ LONG $0xc7048b4a // mov rax, qword [rdi + 8*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB6_3
+
+LBB6_4:
+ RET
+
+TEXT ·_transpose_int64_uint8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB7_1
+
+LBB7_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0x8b48; BYTE $0x17 // mov rdx, qword [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x08578b48 // mov rdx, qword [rdi + 8]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x10578b48 // mov rdx, qword [rdi + 16]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x18578b48 // mov rdx, qword [rdi + 24]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x20c78348 // add rdi, 32
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB7_5
+
+LBB7_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB7_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB7_3:
+ LONG $0xc7048b4a // mov rax, qword [rdi + 8*r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB7_3
+
+LBB7_4:
+ RET
+
+TEXT ·_transpose_uint8_int8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB8_1
+
+LBB8_5:
+ WORD $0xd089 // mov eax, edx
+ WORD $0xb60f; BYTE $0x17 // movzx edx, byte [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x0157b60f // movzx edx, byte [rdi + 1]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x0257b60f // movzx edx, byte [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x0357b60f // movzx edx, byte [rdi + 3]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x04c78348 // add rdi, 4
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB8_5
+
+LBB8_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB8_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB8_3:
+ LONG $0x04b60f42; BYTE $0x07 // movzx eax, byte [rdi + r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB8_3
+
+LBB8_4:
+ RET
+
+TEXT ·_transpose_int8_int8_avx2(SB), $0-32
+
+ MOVQ src+0(FP), DI
+ MOVQ dest+8(FP), SI
+ MOVQ length+16(FP), DX
+ MOVQ transposeMap+24(FP), CX
+
+ WORD $0xfa83; BYTE $0x04 // cmp edx, 4
+ JL LBB9_1
+
+LBB9_5:
+ WORD $0xd089 // mov eax, edx
+ LONG $0x17be0f48 // movsx rdx, byte [rdi]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x1688 // mov byte [rsi], dl
+ LONG $0x57be0f48; BYTE $0x01 // movsx rdx, byte [rdi + 1]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x01 // mov byte [rsi + 1], dl
+ LONG $0x57be0f48; BYTE $0x02 // movsx rdx, byte [rdi + 2]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x02 // mov byte [rsi + 2], dl
+ LONG $0x57be0f48; BYTE $0x03 // movsx rdx, byte [rdi + 3]
+ LONG $0x9114b60f // movzx edx, byte [rcx + 4*rdx]
+ WORD $0x5688; BYTE $0x03 // mov byte [rsi + 3], dl
+ WORD $0x508d; BYTE $0xfc // lea edx, [rax - 4]
+ LONG $0x04c78348 // add rdi, 4
+ LONG $0x04c68348 // add rsi, 4
+ WORD $0xf883; BYTE $0x07 // cmp eax, 7
+ JG LBB9_5
+
+LBB9_1:
+ WORD $0xd285 // test edx, edx
+ JLE LBB9_4
+ WORD $0xc283; BYTE $0x01 // add edx, 1
+ WORD $0x3145; BYTE $0xc0 // xor r8d, r8d
+
+LBB9_3:
+ LONG $0x04be0f4a; BYTE $0x07 // movsx rax, byte [rdi + r8]
+ LONG $0x8104b60f // movzx eax, byte [rcx + 4*rax]
+ LONG $0x06048842 // mov byte [rsi + r8], al
+ LONG $0x01c08349 // add r8, 1
+ WORD $0xc283; BYTE $0xff // add edx, -1
+ WORD $0xfa83; BYTE $0x01 // cmp edx, 1
+ JG LBB9_3
+
+LBB9_4:
+ RET
+
... 6774 lines suppressed ...