You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2026/08/28 21:34:07 UTC
(arrow-go) branch main updated: perf(arrow/array): batch dictionary empty appends (#1223)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new a0dcf383 perf(arrow/array): batch dictionary empty appends (#1223)
a0dcf383 is described below
commit a0dcf3837183f48ce856538b90f70d4e59f76db5
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Aug 28 23:34:02 2026 +0200
perf(arrow/array): batch dictionary empty appends (#1223)
## What does this change?
- `AppendEmptyValues` used to build an `n`-element values array and
dictionary-encode every empty value.
- Resolve the empty value once, reserve the index builder once, and
append the same dictionary index for the whole batch.
- Keep the existing null dictionary path unchanged.
## Benchmark
Command:
```text
go test ./arrow/array -run '^$' -bench '^BenchmarkDictionaryBuilderAppendEmptyValues$' -benchmem -benchtime=200ms -count=3
```
Apple M1 Pro, Go 1.26.3. The benchmark reuses the dictionary builder and
resets the output length between iterations.
| Type | Count | Before ns/op | After ns/op | Before B/op | After B/op |
| --- | ---: | ---: | ---: | ---: | ---: |
| int32 | 64 | 1,684 | 653 | 1,168 | 784 |
| int32 | 4,096 | 81,910 | 14,193 | 42,576 | 784 |
| int32 | 65,536 | 1,305,061 | 221,358 | 551,382 | 784 |
| string | 64 | 2,573 | 868 | 1,568 | 1,056 |
| string | 4,096 | 121,191 | 14,577 | 42,912 | 1,056 |
| string | 65,536 | 1,916,937 | 230,499 | 551,714 | 1,056 |
## Tests
- `go test ./arrow/array -count=1`
- `go test ./arrow/... ./internal/...`
---
arrow/array/dictionary.go | 11 ++-
...ictionary_append_empty_values_benchmark_test.go | 64 ++++++++++++++++++
arrow/array/dictionary_append_empty_values_test.go | 78 ++++++++++++++++++++++
3 files changed, 151 insertions(+), 2 deletions(-)
diff --git a/arrow/array/dictionary.go b/arrow/array/dictionary.go
index db5c9422..fe7d66be 100644
--- a/arrow/array/dictionary.go
+++ b/arrow/array/dictionary.go
@@ -876,13 +876,20 @@ func (b *dictionaryBuilder) AppendEmptyValues(n int) {
valueBuilder := NewBuilder(b.mem, b.dt.ValueType)
defer valueBuilder.Release()
- valueBuilder.AppendEmptyValues(n)
+ valueBuilder.AppendEmptyValue()
values := valueBuilder.NewArray()
defer values.Release()
- if err := b.AppendArray(values); err != nil {
+ idx, _, err := b.memoTable.GetOrInsert(getvalFn(values)(0))
+ if err != nil {
panic(err)
}
+
+ b.idxBuilder.Reserve(n)
+ for i := 0; i < n; i++ {
+ b.idxBuilder.UnsafeAppend(idx)
+ }
+ b.length += n
}
func (b *dictionaryBuilder) UnsafeAppendBoolToBitmap(v bool) {
diff --git a/arrow/array/dictionary_append_empty_values_benchmark_test.go b/arrow/array/dictionary_append_empty_values_benchmark_test.go
new file mode 100644
index 00000000..af5056d8
--- /dev/null
+++ b/arrow/array/dictionary_append_empty_values_benchmark_test.go
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+func BenchmarkDictionaryBuilderAppendEmptyValues(b *testing.B) {
+ valueTypes := []struct {
+ name string
+ typ arrow.DataType
+ }{
+ {"int32", arrow.PrimitiveTypes.Int32},
+ {"string", arrow.BinaryTypes.String},
+ }
+
+ for _, valueType := range valueTypes {
+ valueType := valueType
+ b.Run(valueType.name, func(b *testing.B) {
+ for _, count := range []int{1, 64, 4096, 65536} {
+ count := count
+ b.Run(fmt.Sprintf("count-%d", count), func(b *testing.B) {
+ mem := memory.NewGoAllocator()
+ bldr := array.NewDictionaryBuilder(mem, &arrow.DictionaryType{
+ IndexType: arrow.PrimitiveTypes.Int32,
+ ValueType: valueType.typ,
+ })
+ defer bldr.Release()
+
+ bldr.AppendEmptyValues(count)
+ bldr.Resize(0)
+
+ b.ReportAllocs()
+ b.SetBytes(int64(count))
+ b.ResetTimer()
+ for b.Loop() {
+ bldr.Resize(0)
+ bldr.AppendEmptyValues(count)
+ }
+ })
+ }
+ })
+ }
+}
diff --git a/arrow/array/dictionary_append_empty_values_test.go b/arrow/array/dictionary_append_empty_values_test.go
new file mode 100644
index 00000000..c45eee51
--- /dev/null
+++ b/arrow/array/dictionary_append_empty_values_test.go
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestInt32DictionaryBuilderAppendEmptyValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: &arrow.Int8Type{}, ValueType: arrow.PrimitiveTypes.Int32}
+ bldr := array.NewDictionaryBuilder(mem, dictType)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValues(0)
+ bldr.AppendEmptyValues(-1)
+ assert.Equal(t, 0, bldr.Len())
+
+ bldr.AppendEmptyValues(8)
+ result := bldr.NewDictionaryArray()
+ defer result.Release()
+
+ dict := result.Dictionary().(*array.Int32)
+ assert.Equal(t, 1, dict.Len())
+ assert.Equal(t, int32(0), dict.Value(0))
+ assert.Equal(t, 8, result.Len())
+ assert.Equal(t, 0, result.NullN())
+ for i := 0; i < result.Len(); i++ {
+ assert.False(t, result.IsNull(i))
+ assert.Equal(t, 0, result.GetValueIndex(i))
+ }
+}
+
+func TestNullDictionaryBuilderAppendEmptyValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: &arrow.Int8Type{}, ValueType: arrow.Null}
+ bldr := array.NewDictionaryBuilder(mem, dictType)
+ defer bldr.Release()
+
+ bldr.AppendEmptyValues(0)
+ bldr.AppendEmptyValues(-1)
+ assert.Equal(t, 0, bldr.Len())
+
+ bldr.AppendEmptyValues(8)
+ result := bldr.NewDictionaryArray()
+ defer result.Release()
+
+ require.Equal(t, 8, result.Len())
+ assert.Equal(t, 8, result.NullN())
+ assert.Equal(t, 0, result.Dictionary().Len())
+ for i := 0; i < result.Len(); i++ {
+ assert.True(t, result.IsNull(i))
+ }
+}