(iceberg-go) branch main updated: fix(transforms): truncate strings by Unicode code points instead of UTF-8 bytes (#1223)

[at...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new ae7b4667 fix(transforms): truncate strings by Unicode code points instead of UTF-8 bytes (#1223)
ae7b4667 is described below

commit ae7b466771d75cf7c60fee6998e2af1999783e10
Author: Badal Prasad Singh <ba...@datazip.io>
AuthorDate: Thu Jun 18 04:24:28 2026 +0530

    fix(transforms): truncate strings by Unicode code points instead of UTF-8 bytes (#1223)
    
    ## Problem
    
    Iceberg-Go's `iceberg.TruncateTransform.Transformer` for `StringType`
    slices the input by raw UTF-8 byte length instead of Unicode code
    points. For any non-ASCII input — CJK, emoji, accented Latin — this
    produces output that may not be valid UTF-8.
    
    Following the [**Iceberg
    spec**](https://iceberg.apache.org/spec/#truncate-transform-details)
    defining string truncate as truncation to *N Unicode characters*.
    
    
    Fixes #1222
    
    ## Solution
    
    Iterate the input by Unicode code points using `unicode/utf8`. Keep
    binary truncation byte-based.
    
    ## Files changed
    
    | File | Description |
    |---|---|
    | `transforms.go` | Bug Fix |
    | `transforms_test.go` | Regression Test |
    
    ## Testing
    - Ingested data into a partitioned iceberg table in Glue Catalog
    - Queried through Trino
    
    ---------
    
    Signed-off-by: badalprasadsingh <ba...@datazip.io>
---
 transforms.go      |  9 +++++++-
 transforms_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/transforms.go b/transforms.go
index a274f572..85293966 100644
--- a/transforms.go
+++ b/transforms.go
@@ -27,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 	"unsafe"
 
 	"github.com/apache/arrow-go/v18/arrow/decimal128"
@@ -447,7 +448,13 @@ func (t TruncateTransform) Transformer(src Type) (func(any) any, error) {
 		return func(v any) any {
 			switch v := v.(type) {
 			case string:
-				return v[:min(len(v), t.Width)]
+				byteOff := 0
+				for cp := 0; cp < t.Width && byteOff < len(v); cp++ {
+					_, size := utf8.DecodeRuneInString(v[byteOff:])
+					byteOff += size
+				}
+
+				return v[:byteOff]
 			case []byte:
 				return v[:min(len(v), t.Width)]
 			default:
diff --git a/transforms_test.go b/transforms_test.go
index 223ba96d..22e8a2c2 100644
--- a/transforms_test.go
+++ b/transforms_test.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"unicode/utf8"
 
 	"github.com/apache/arrow-go/v18/arrow/decimal"
 	"github.com/apache/arrow-go/v18/arrow/decimal128"
@@ -716,3 +717,63 @@ func TestTruncateTransform(t *testing.T) {
 		})
 	}
 }
+
+func TestTruncateTransformStringUnicode(t *testing.T) {
+	tests := []struct {
+		name     string
+		width    int
+		input    string
+		expected string
+	}{
+		{
+			name:     "truncate_unicode_string_width_2",
+			width:    2,
+			input:    "イロハニホヘト",
+			expected: "イロ",
+		},
+		{
+			name:     "truncate_unicode_string_width_3",
+			width:    3,
+			input:    "イロハニホヘト",
+			expected: "イロハ",
+		},
+		{
+			name:     "truncate_multibyte_string_width_1",
+			width:    1,
+			input:    "测试",
+			expected: "测",
+		},
+		{
+			name:     "truncate_mixed_unicode_and_ascii",
+			width:    4,
+			input:    "测试raul试测",
+			expected: "测试ra",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			transform := iceberg.TruncateTransform{Width: tt.width}
+
+			fn, err := transform.Transformer(iceberg.PrimitiveTypes.String)
+			require.NoError(t, err)
+
+			gotAny := fn(tt.input)
+			got, ok := gotAny.(string)
+			require.True(t, ok, "Transformer must return string for StringType")
+			assert.Equal(t, tt.expected, got, "Transformer truncated value must match Java's code-point semantics")
+			assert.True(t, utf8.ValidString(got),
+				"Transformer output must be valid UTF-8; got bytes %x for input %q",
+				got, tt.input)
+
+			result := transform.Apply(iceberg.Optional[iceberg.Literal]{
+				Val: iceberg.StringLiteral(tt.input), Valid: true,
+			})
+			require.True(t, result.Valid)
+			assert.Equal(t, iceberg.StringLiteral(tt.expected), result.Val,
+				"Apply on StringLiteral must match Transformer semantics")
+			assert.True(t, utf8.ValidString(string(result.Val.(iceberg.StringLiteral))),
+				"Apply output must be valid UTF-8")
+		})
+	}
+}