You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "serramatutu (via GitHub)" <gi...@apache.org> on 2025/10/30 11:16:27 UTC

[PR] [DRAFT] Add `TimestampWithOffset` extension type [arrow-go]

serramatutu opened a new pull request, #558:
URL: https://github.com/apache/arrow-go/pull/558

   # Which issue does this PR close?
   
   No issue yet. This is still being discussed in the mailing list, and this PR is here for reference.
   
   # Rationale for this change
   
   Please refer to the mailing list discussion.
   
   # What changes are included in this PR?
   
   This commit adds a new `TimestampWithOffset` extension type. This type represents a timestamp column that stores potentially different timezone offsets per value. The timestamp is stored in UTC alongside the original timezone offset in minutes.
   
   # Are these changes tested?
   
   Yes.
   
   # Are there any user-facing changes?
   
   Yes, this is a new proposed canonical extension type.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3562127195


##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,591 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var epoch = time.Unix(0, 0).In(time.FixedZone("UTC+00:00", 0))
+
+var testDate0 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:30", -8*60*60-30*60)
+var testDate1 = testDate0.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+11:00", +11*60*60)
+var testDate2 = testDate0.In(testZone2)
+
+func dict(index arrow.DataType) arrow.DataType {
+	return &arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+}
+
+func ree(runEnds arrow.DataType) arrow.DataType {
+	v := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	v.ValueNullable = false
+	return v
+}
+
+// All tests use this in a for loop to make sure everything works for every possible
+// encoding of offsets (primitive, dictionary, run-end)
+var allAllowedOffsetTypes = make(map[string]arrow.DataType)
+
+func init() {
+	// primitive offsetType
+	allAllowedOffsetTypes["primitive-int16"] = arrow.PrimitiveTypes.Int16
+
+	// dict-encoded offsetType
+	allAllowedOffsetTypes["dict-Uint8"] = dict(arrow.PrimitiveTypes.Uint8)
+	allAllowedOffsetTypes["dict-Uint16"] = dict(arrow.PrimitiveTypes.Uint16)
+	allAllowedOffsetTypes["dict-Uint32"] = dict(arrow.PrimitiveTypes.Uint32)
+	allAllowedOffsetTypes["dict-Uint64"] = dict(arrow.PrimitiveTypes.Uint64)
+	allAllowedOffsetTypes["dict-Int8"] = dict(arrow.PrimitiveTypes.Int8)
+	allAllowedOffsetTypes["dict-Int16"] = dict(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["dict-Int32"] = dict(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["dict-Int64"] = dict(arrow.PrimitiveTypes.Int64)
+
+	// run-end encoded offsetType
+	allAllowedOffsetTypes["ree-Int16"] = ree(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["ree-Int32"] = ree(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["ree-Int64"] = ree(arrow.PrimitiveTypes.Int64)
+}
+
+func TestTimestampWithOffsetTypePrimitiveBasics(t *testing.T) {
+	typ := extensions.NewTimestampWithOffsetType(testTimeUnit)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     arrow.PrimitiveTypes.Int16,
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func assertDictBasics[I extensions.DictIndexType](t *testing.T, indexType I) {
+	typ := extensions.NewTimestampWithOffsetTypeDictionaryEncoded(testTimeUnit, indexType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     dict(arrow.DataType(indexType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeDictionaryEncodedBasics(t *testing.T) {
+	assertDictBasics(t, &arrow.Uint8Type{})
+	assertDictBasics(t, &arrow.Uint16Type{})
+	assertDictBasics(t, &arrow.Uint32Type{})
+	assertDictBasics(t, &arrow.Uint64Type{})
+	assertDictBasics(t, &arrow.Int8Type{})
+	assertDictBasics(t, &arrow.Int16Type{})
+	assertDictBasics(t, &arrow.Int32Type{})
+	assertDictBasics(t, &arrow.Int64Type{})
+}
+
+func assertReeBasics[E extensions.RunEndsType](t *testing.T, runEndsType E) {
+	typ := extensions.NewTimestampWithOffsetTypeRunEndEncoded(testTimeUnit, runEndsType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     ree(arrow.DataType(runEndsType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeRunEndEncodedBasics(t *testing.T) {
+	assertReeBasics(t, &arrow.Int16Type{})
+	assertReeBasics(t, &arrow.Int32Type{})
+	assertReeBasics(t, &arrow.Int64Type{})
+}
+
+func TestTimestampWithOffsetEquals(t *testing.T) {
+	// Completely different types are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewBool8Type()))
+
+	// Different time units are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+
+	// Different underlying storage type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+
+	// Dict-encoding key type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+
+	// REE index type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+
+	// Equals OK
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Nanosecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+}
+
+func TestTimestampWithOffsetExtensionBuilder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, _ := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+
+			builder.Append(testDate0)
+			builder.AppendNull()
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			builder.Append(epoch)
+
+			// it should build the array with the correct size
+			arr := builder.NewArray()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+			assert.Equal(t, 5, arr.Data().Len())
+			defer arr.Release()
+
+			// typedArr.Value(i) should return values adjusted for their original timezone
+			assert.Equal(t, testDate0, typedArr.Value(0))
+			assert.Equal(t, testDate1, typedArr.Value(2))
+			assert.Equal(t, testDate2, typedArr.Value(3))
+			assert.Equal(t, epoch, typedArr.Value(4))
+
+			// storage TimeUnit should be the same as we pass in to the builder, and storage timezone should be UTC
+			timestampStructField := typedArr.Storage().(*array.Struct).Field(0)
+			timestampStructDataType := timestampStructField.DataType().(*arrow.TimestampType)
+			assert.Equal(t, timestampStructDataType.Unit, testTimeUnit)
+			assert.Equal(t, timestampStructDataType.TimeZone, "UTC")
+
+			// stored values should be equivalent to the raw values in UTC
+			timestampsArr := timestampStructField.(*array.Timestamp)
+			assert.Equal(t, testDate0.In(time.UTC), timestampsArr.Value(0).ToTime(testTimeUnit))
+			assert.Equal(t, testDate1.In(time.UTC), timestampsArr.Value(2).ToTime(testTimeUnit))
+			assert.Equal(t, testDate2.In(time.UTC), timestampsArr.Value(3).ToTime(testTimeUnit))
+			assert.Equal(t, epoch.In(time.UTC), timestampsArr.Value(4).ToTime(testTimeUnit))
+
+			// the array should encode itself as JSON and string
+			arrStr := arr.String()
+			assert.Equal(t, fmt.Sprintf(`["%[1]s" (null) "%[2]s" "%[3]s" "%[4]s"]`, testDate0, testDate1, testDate2, epoch), arrStr)
+			jsonStr, err := json.Marshal(arr)
+			assert.NoError(t, err)
+
+			// roundtripping from JSON with array.FromJSON should work
+			expectedDataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			roundtripped, _, err := array.FromJSON(mem, expectedDataType, bytes.NewReader(jsonStr))
+			defer roundtripped.Release()
+			assert.NoError(t, err)
+			assert.Truef(t, array.Equal(arr, roundtripped), "expected %s\n\ngot %s", arr, roundtripped)
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesLeadingNull(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			// A leading null as the very first operation on a fresh builder must not
+			// corrupt the offsets. Reading every value below would panic on a
+			// run-end encoded array whose run-ends and values children fell out of
+			// sync (which happened when a run was continued before one existed).
+			builder.AppendValues([]time.Time{epoch, testDate1, testDate2}, []bool{false, true, true})
+
+			arr := builder.NewArray()
+			defer arr.Release()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+			require.Equal(t, 3, arr.Data().Len())
+			assert.True(t, typedArr.IsNull(0))
+			assert.Equal(t, testDate1, typedArr.Value(1))
+			assert.Equal(t, testDate2, typedArr.Value(2))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderRunEndEncodedNullContinuesRun(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// The same offset surrounds a null. The null must continue the current run
+	// rather than reset the tracked offset, otherwise one logical run is split
+	// into two adjacent runs that share the same value.
+	builder.AppendValues([]time.Time{testDate1, epoch, testDate1}, []bool{true, false, true})
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	offsets := typedArr.Storage().(*array.Struct).Field(1).(*array.RunEndEncoded)
+	assert.Equal(t, 1, offsets.Values().Len())
+
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesNilValids(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// nil and empty validity slices must both be treated as all-valid so the
+	// parent struct and its child builders stay the same length.
+	validsVariants := map[string][]bool{"nil": nil, "empty": {}}
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		for validsName, valids := range validsVariants {
+			t.Run(name+"/"+validsName, func(t *testing.T) {
+				builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+				require.NoError(t, err)
+
+				builder.AppendValues([]time.Time{testDate0, testDate1, testDate2}, valids)
+
+				arr := builder.NewArray()
+				defer arr.Release()
+				typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+				require.Equal(t, 3, arr.Data().Len())
+				assert.Equal(t, testDate0, typedArr.Value(0))
+				assert.Equal(t, testDate1, typedArr.Value(1))
+				assert.Equal(t, testDate2, typedArr.Value(2))
+			})
+		}
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesMismatchedValidsPanics(t *testing.T) {
+	builder, err := extensions.NewTimestampWithOffsetBuilder(memory.DefaultAllocator, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	defer builder.Release()
+
+	assert.Panics(t, func() {
+		builder.AppendValues([]time.Time{testDate0, testDate1}, []bool{true})
+	})
+}
+
+func TestTimestampWithOffsetBuilderReuseAfterNewArray(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			builder.Append(testDate1)
+			arr1 := builder.NewArray()
+			defer arr1.Release()
+
+			// Reusing the builder must start a fresh run even when the first value
+			// repeats the previous array's offset; otherwise a run-end-encoded
+			// offset is continued on a freshly reset (empty) builder.
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			arr2 := builder.NewArray()
+			defer arr2.Release()
+
+			typedArr := arr2.(*extensions.TimestampWithOffsetArray)
+			require.Equal(t, 2, arr2.Data().Len())
+			assert.Equal(t, testDate1, typedArr.Value(0))
+			assert.Equal(t, testDate2, typedArr.Value(1))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendNullBetweenEqualOffsets(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// AppendNull appends a null offset run; the following value repeats the
+	// pre-null offset and must start its own run instead of being folded into
+	// the null run (which would decode the trailing value's offset as null).
+	builder.Append(testDate1)
+	builder.AppendNull()
+	builder.Append(testDate1)
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	require.Equal(t, 3, arr.Data().Len())
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetRunEndEncodedSliced(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for _, runEnds := range []arrow.DataType{
+		arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64,
+	} {
+		t.Run(runEnds.String(), func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(runEnds))
+			require.NoError(t, err)
+
+			// Offsets [0,0,-510,-510,-510,660,0] give run-ends [2,5,6,7], so a
+			// slice starting at index 3 begins inside a later run: the case where
+			// an offset-unaware physical index decodes the wrong run.
+			values := []time.Time{testDate0, testDate0, testDate1, testDate1, testDate1, testDate2, epoch}
+			builder.AppendValues(values, nil)
+			arr := builder.NewArray().(*extensions.TimestampWithOffsetArray)
+			defer arr.Release()
+
+			want := arr.Values()
+
+			for start := 0; start < arr.Len(); start++ {
+				for end := start; end <= arr.Len(); end++ {
+					sliced := array.NewSlice(arr, int64(start), int64(end)).(*extensions.TimestampWithOffsetArray)
+					got := sliced.Values()
+					require.Equalf(t, end-start, len(got), "slice [%d:%d]", start, end)
+					for k := 0; k < end-start; k++ {
+						require.Truef(t, want[start+k].Equal(got[k]), "slice [%d:%d] row %d instant: want %s got %s", start, end, k, want[start+k], got[k])
+						_, wantOff := want[start+k].Zone()
+						_, gotOff := got[k].Zone()
+						require.Equalf(t, wantOff, gotOff, "slice [%d:%d] row %d offset", start, end, k)
+					}
+					sliced.Release()
+				}
+			}
+		})
+	}
+}
+
+func TestTimestampWithOffsetSubHourNegativeZone(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// A negative offset smaller than one hour must keep its sign in the zone
+	// name (regression: -30 minutes formatted as "UTC+00:30" instead of "UTC-00:30").
+	b, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	b.Append(testDate0.In(time.FixedZone("UTC-00:30", -30*60)))
+	arr := b.NewArray().(*extensions.TimestampWithOffsetArray)
+	defer arr.Release()
+
+	name, off := arr.Value(0).Zone()
+	require.Equal(t, -30*60, off)
+	require.Equal(t, "UTC-00:30", name)
+}
+
+func TestTimestampWithOffsetExtensionRecordBuilder(t *testing.T) {
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			dataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			schema := arrow.NewSchema([]arrow.Field{
+				{
+					Name:     "timestamp_with_offset",
+					Nullable: true,
+					Type:     dataType,
+				},
+			}, nil)
+			builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
+			defer builder.Release()
+
+			fieldBuilder := builder.Field(0).(*extensions.TimestampWithOffsetBuilder)
+
+			// append a simple time.Time
+			fieldBuilder.Append(testDate0)
+
+			// append the epoch
+			fieldBuilder.Append(epoch)
+
+			// append a null and 2 time.Time all at once
+			values := []time.Time{
+				time.Unix(0, 0).In(time.UTC),
+				testDate1,
+				testDate2,
+			}
+			valids := []bool{false, true, true}
+			fieldBuilder.AppendValues(values, valids)
+
+			// append a value from RFC3339 string
+			fieldBuilder.AppendValueFromString(testDate0.Format(time.RFC3339))
+
+			// append value formatted in a different string layout
+			fieldBuilder.Layout = time.RFC3339Nano
+			fieldBuilder.AppendValueFromString(testDate1.Format(time.RFC3339Nano))
+
+			record := builder.NewRecordBatch()
+
+			// Record batch should JSON-encode values containing per-row timezone info
+			json, err := record.MarshalJSON()
+			require.NoError(t, err)
+			expect := `[{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"1970-01-01T00:00:00Z"}
+,{"timestamp_with_offset":null}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+,{"timestamp_with_offset":"2025-01-01T11:00:00+11:00"}
+,{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+]`
+			require.Equal(t, expect, string(json))
+
+			// Record batch roundtrip to JSON should work
+			roundtripped, _, err := array.RecordFromJSON(memory.DefaultAllocator, schema, bytes.NewReader(json))
+			require.NoError(t, err)
+			defer roundtripped.Release()
+			require.Equal(t, schema, roundtripped.Schema())
+			// Avoid array.RecordEqual here: it compares the inner non-nullable

Review Comment:
   Can we add a FIXME here to revisit? Ideally the records SHOULD equal after roundtripping lol



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #558:
URL: https://github.com/apache/arrow-go/pull/558#issuecomment-4929133223

   @serramatutu filed #918 to split out the nullability stuff


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085514700


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type

Review Comment:
   Actually I had to revert. Spec says those types are different:
   
   Dict index is either signed or unsigned int of 8-64 bits
   > The index type of a Dictionary type can only be an integer type, preferably signed, with width 8 to 64 bits.
   
   Run ends is a signed int of 16-64 bits
   > The run end type of a Run-End Encoded type can only be a signed integer type with width 16 to 64 bits.
   
   https://arrow.apache.org/docs/format/Columnar.html#data-types



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746081348


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   Oops, yea this was reminiscent of the old implementation before we all circled around to this being non-nullable again...



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746048830


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758798540


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   @felipecrv Tells me `UnsafeAppendBoolToBitmap()` should not be callable on dictionary/REE. So I should probably revert the commit I mentioned above and just call `AppendNulls()`.
   
   Either way, I think it would be nice to either:
   - immediately panic when this gets called on Dict/REE
   - just make it call `AppendNulls()` and ignore the fact there is no bitmap
   
   otherwise people will get very strange errors when creating a `RecordBatch` (like I did).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3561126196


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}

Review Comment:
   Fixed in e7ef6d8d: the zone-name sign is now derived from the whole offset, so a sub-hour negative offset keeps its sign (e.g. -30 minutes now formats as "UTC-00:30" instead of "UTC+00:30"). Added TestTimestampWithOffsetSubHourNegativeZone as a regression test.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	_, offsetSeconds := t.Zone()
+	offsetMinutes := int16(offsetSeconds / 60)
+	ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
+	return ts, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Time{}
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// The second parameter indicates whether the timestamp is valid or not.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq2[time.Time, bool] {
+	return func(yield func(time.Time, bool) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			// Run-ends are absolute over the unsliced offsets child, so a sliced
+			// array must begin at the physical run covering its logical offset and
+			// advance using absolute positions (logicalOffset + i).
+			logicalOffset := reeOffsets.Offset()
+			offsetPhysicalIdx := reeOffsets.GetPhysicalOffset()
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if logicalOffset+i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	return slices.Collect(func(yield func(time.Time) bool) {
+		for time := range a.iterValues() {
+			if !yield(time) {
+				return
+			}
+		}
+	})
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	i := 0
+	for ts, valid := range a.iterValues() {
+		if !valid {
+			values[i] = nil
+		} else {
+			values[i] = &ts
+		}
+		i += 1
+	}
+	return json.Marshal(values)
+}

Review Comment:
   Addressed in e7ef6d8d: MarshalJSON now stores the timestamp by value instead of the address of the range variable. (For the record, under this module's Go 1.25 per-iteration loop semantics it was not actually an aliasing bug — the existing exact-JSON roundtrip test asserts distinct per-row values — but marshaling by value is clearer.)



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	_, offsetSeconds := t.Zone()
+	offsetMinutes := int16(offsetSeconds / 60)
+	ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
+	return ts, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Time{}
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// The second parameter indicates whether the timestamp is valid or not.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq2[time.Time, bool] {
+	return func(yield func(time.Time, bool) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			// Run-ends are absolute over the unsliced offsets child, so a sliced
+			// array must begin at the physical run covering its logical offset and
+			// advance using absolute positions (logicalOffset + i).
+			logicalOffset := reeOffsets.Offset()
+			offsetPhysicalIdx := reeOffsets.GetPhysicalOffset()
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if logicalOffset+i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	return slices.Collect(func(yield func(time.Time) bool) {
+		for time := range a.iterValues() {
+			if !yield(time) {
+				return
+			}
+		}
+	})
+}

Review Comment:
   Fixed in e7ef6d8d: renamed the loop variable to `t` so it no longer shadows the `time` package.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746081348


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   Oops, yea this was reminiscent of the old implementation before we all circled around to this being non-nullable again... Good catch :)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746910074


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {

Review Comment:
   I like it. I prefer compile-time errors over runtime errors :)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated: I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap()` from the `Builder` interface. The default implementation does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state where the parent has greater length than the index builder. This causes a panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? Also not sure if my change is good enough since it's very easy to forget to override `UnsafeAppendBoolToBitmap()`.
   
    https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758798540


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   @felipecrv Tells me `UnsafeAppendBoolToBitmap()` should not be callable on dictionary/REE. So I should probably revert the commit I mentioned above and just call `AppendNulls()`.
   
   Either way, I think it would be nice to immediately panic when this gets called on Dict/REE, otherwise people will get very strange errors when creating a `RecordBatch` (like I did).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "lidavidm (via GitHub)" <gi...@apache.org>.
lidavidm commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3608144452


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,608 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable

Review Comment:
   This is pretty major right? Can we link this to a tracking issue? (Does integration catch this?)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758798540


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   @felipecrv Tells me `UnsafeAppendBoolToBitmap()` should not be callable on dictionary/REE. So I'll actually revert the commit I mentioned above and just call `AppendNulls()`.
   
   Either way, I think it would be nice to immediately panic when this gets called on Dict/REE, otherwise people will get very strange errors when creating a `RecordBatch`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746047798


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)

Review Comment:
   I ended up not being able to do this because I changed `iterValues()` to return `time.Time` instead of `*time.Time`. So we need a check if `time.Unix() == 0`, meaning it should actually be serialized as `null` instead of `1970-01-01`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746054571


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   This is not semantically equal (I tried, it messes up the values, but only sometimes for some reason?).
   
   What I'm trying to accomplish here is get the `naiveUtc` by converting all the "raw" values without actually going through Go's timezone conversion logic, i.e simply dropping the timezone without any conversion of the underlying values. Then I apply my own timezone based on the offset.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #558:
URL: https://github.com/apache/arrow-go/pull/558#issuecomment-4928853326

   @serramatutu updated this PR to be *only* the timestamp with offset extension type


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085965193


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)
+			}
+		}
+	case *array.Int16DictionaryBuilder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.Append(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   I did change this but now the IPC roundtrip tests are failing because the internal count of nulls is different between the thing before IPC and after IPC.
   
   To make them pass we'll need to relax the `arrow.RecordEqual()` to ignore `NullN()` when the field is not nullable.
   
   Related: https://lists.apache.org/thread/7gbqjwykh1ob3xbvwph3ljsdl5c7kxpd



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746886513


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   As we discussed in Slack, I found an issue when roundtripping to JSON. 
   
   It looks like replacing these with `UnsafeAppend` results in an array with `nulls=0` in the inner (non-nullable) field.
   If I write the same array to JSON and parse it back with `arrow.RecordFromJSON()`, the inner array has `nulls=1`, even if the field is not nullable.
   
   This leads to `arrow.RecordEqual()` returning `false` due to `original.NullN() != roundtrip.NullN()`.
   
   There is an ongoing thread on the mainling list about how to resolve this, so I'll hold off on fixing it for now.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746875996


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/e36a78ffad1d8c650a75e9adf12312caec99a5c6



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758798540


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   @felipecrv Tells me `UnsafeAppendBoolToBitmap()` should not be callable on dictionary/REE. So I should probably revert the commit I mentioned above and just call `AppendNulls()`.
   
   Either way, I think it would be nice to immediately panic when this gets called on Dict/REE, otherwise people will get very strange errors when creating a `RecordBatch`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3609577570


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,608 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable

Review Comment:
   This is part of what was pulled out into #918 and friends



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085965193


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)
+			}
+		}
+	case *array.Int16DictionaryBuilder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.Append(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   I did change this but now the IPC roundtrip tests are failing because the internal count of nulls is different between the thing before IPC and after IPC.
   
   To make them pass we'll need to relax the `arrow.RecordEqual()` to ignore `NullN()` when the field is not nullable. Alternatively we can change it so that an array builder sets `nulls = 0` if the array is not nullable, regardless of what has been set before in calls to `UnsafeAppendBoolToBitmap()` etc
   
   Related: https://lists.apache.org/thread/7gbqjwykh1ob3xbvwph3ljsdl5c7kxpd



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3523587531


##########
arrow/array/compare.go:
##########
@@ -35,9 +39,17 @@ func RecordEqual(left, right arrow.RecordBatch) bool {
 	}
 
 	for i := range left.Columns() {
+		lf := left.Schema().Field(i)
+		rf := left.Schema().Field(i)

Review Comment:
   Addressed in the current head: `recordEqual` now takes `rf := right.Schema().Field(i)` and gates on `schemaFieldEqual(lf, rf)` (name + nullability), so left/right field differences are detected. Resolving.



##########
arrow/array/compare.go:
##########
@@ -47,16 +59,26 @@ func RecordEqual(left, right arrow.RecordBatch) bool {
 // RecordApproxEqual reports whether the two provided records are approximately equal.
 // For non-floating point columns, it is equivalent to RecordEqual.
 func RecordApproxEqual(left, right arrow.RecordBatch, opts ...EqualOption) bool {
+	return recordApproxEqual(left, right, newEqualOption(opts...))
+}
+
+func recordApproxEqual(left, right arrow.RecordBatch, opt equalOption) bool {
 	switch {
 	case left.NumCols() != right.NumCols():
 		return false
 	case left.NumRows() != right.NumRows():
 		return false
 	}
 
-	opt := newEqualOption(opts...)
-
 	for i := range left.Columns() {
+		lf := left.Schema().Field(i)
+		rf := left.Schema().Field(i)

Review Comment:
   Addressed: `recordApproxEqual` now reads `rf := right.Schema().Field(i)` and checks `schemaFieldEqual(lf, rf)` before comparing, so nullability/name differences are caught. Resolving.



##########
arrow/array/compare.go:
##########
@@ -160,21 +194,29 @@ func TableEqual(left, right arrow.Table) bool {
 	}
 
 	for i := 0; int64(i) < left.NumCols(); i++ {
-		lc := left.Column(i)
-		rc := right.Column(i)
-		if !lc.Field().Equal(rc.Field()) {
+		lf := left.Schema().Field(i)
+		rf := left.Schema().Field(i)

Review Comment:
   Addressed: `tableEqual` now uses `rf := right.Schema().Field(i)` with a `schemaFieldEqual(lf, rf)` guard. Resolving.



##########
arrow/array/compare.go:
##########
@@ -183,27 +225,35 @@ func TableApproxEqual(left, right arrow.Table, opts ...EqualOption) bool {
 	}
 
 	for i := 0; int64(i) < left.NumCols(); i++ {
-		lc := left.Column(i)
-		rc := right.Column(i)
-		if !lc.Field().Equal(rc.Field()) {
+		lf := left.Schema().Field(i)
+		rf := left.Schema().Field(i)

Review Comment:
   Addressed: `tableApproxEqual` now uses `rf := right.Schema().Field(i)` with the same `schemaFieldEqual(lf, rf)` guard. Resolving.



##########
arrow/array/record_test.go:
##########
@@ -511,14 +516,20 @@ func TestRecordBuilder(t *testing.T) {
 		}
 	}
 
+	err := b.UnmarshalJSON([]byte(`{"f1-i32": 6, "f2-f64-notnull": 6.6, "map": [{"key": "4": "value": "d"}]}`))

Review Comment:
   Fixed: the literal now reads `{"key": "4", "value": "d"}` (valid JSON) instead of the malformed colon, so the test exercises the intended path. Resolving.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,535 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	_, offsetSeconds := t.Zone()
+	offsetMinutes := int16(offsetSeconds / 60)
+	ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
+	return ts, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Time{}
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// The second parameter indicates whether the timestamp is valid or not.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq2[time.Time, bool] {
+	return func(yield func(time.Time, bool) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			offsetPhysicalIdx := 0
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	return slices.Collect(func(yield func(time.Time) bool) {
+		for time := range a.iterValues() {
+			if !yield(time) {
+				return
+			}
+		}
+	})
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	i := 0
+	for ts, valid := range a.iterValues() {
+		if !valid {
+			values[i] = nil
+		} else {
+			values[i] = &ts
+		}
+		i += 1
+	}
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int, nullable bool) interface{} {
+	if nullable && a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType:       offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for _, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			timestamps.UnsafeAppend(timestamp)
+			offsets.UnsafeAppend(offsetMinutes)
+		}
+	case *array.Int16DictionaryBuilder:
+		for _, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			timestamps.UnsafeAppend(timestamp)
+			offsets.UnsafeAppend(offsetMinutes)
+		}
+	case *array.RunEndEncodedBuilder:
+		offsetValuesBuilder := offsets.ValueBuilder().(*array.Int16Builder)
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			timestamps.UnsafeAppend(timestamp)
+			offsetMinutes16 := int16(offsetMinutes)
+			// If value at i is null, simply continue the run to maximize compression
+			if valids[i] && offsetMinutes != b.lastOffset {
+				offsets.Append(1)
+				offsetValuesBuilder.Append(offsetMinutes16)
+			} else {
+				offsets.ContinueRun(1)
+			}
+			b.lastOffset = offsetMinutes16

Review Comment:
   Fixed in the builder rework. Both `Append` and `AppendValues` now start a run with `offsets.Append(1)` whenever none exists yet (`b.lastOffset == noLastOffset`), so `ContinueRun(1)` is never called without a matching value entry — including leading-null rows. `lastOffset` is only updated on the branch that starts a new run, so a null row can't split a run or shift a boundary based on a masked value, and `AppendNull`/`AppendNulls`/`NewArray` reset the tracker. There's now an inline comment documenting exactly this case. Resolving.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746050075


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2771557250


##########
arrow/array/encoded.go:
##########
@@ -386,6 +386,15 @@ func (b *RunEndEncodedBuilder) ContinueRun(n uint64) {
 	b.addLength(n)
 }
 
+func (b *RunEndEncodedBuilder) UnsafeAppendBoolToBitmap(v bool) {
+	if !v {
+		b.finishRun()
+	}

Review Comment:
   this seems incorrect, it could be a run of nulls, couldn't it? And wouldn't the run itself be handled by the value append? I don't think we need to call `b.finishRun` here



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)

Review Comment:
   `arrow.IsInteger` is unnecessary here, by definition only integer types are allowed for the IndexTypes of a dictionary type. Anything else is invalid anyways, so there's no need to check that.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.

Review Comment:
   Update the comment to remove this since the generics will affirm that we can't have this error



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type

Review Comment:
   this and the `DictIndexType` are both identical, instead of duplicating this we should either create a single type constraint or embed one in the other:
   
   ```go
   type TimestampWithOffsetRunEndsType interface {
        DictIndexType
   }
   ```
   
   My personal preference would be to have a single type though, but I'm not averse to having the two separate ones if necessary.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.

Review Comment:
   remove this line from the comment since the generics prevent this from happening



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)

Review Comment:
   Using `0` as a sentinel for `null` seems incorrect as that means you can't actually encode a non-null value of 1970-01-01 and get the correct output. Since we need to be able to represent the difference between a *value* of `0` and a `null`, then either `iterValues()` needs to return a sequence of `*time.Time` or it needs to be a sequence of `iter.Seq2[time.Time, bool]` where the bool indicates validity (if the bool is false, then it's null).



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   I agree with this, we should update those builders to error / panic if called and add docstring comments to explain that.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E TimestampWithOffsetRunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as TimestampWithOffsetRunEndsType is always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// If the timestamp is null, the returned time will be the unix epoch.

Review Comment:
   what happens if the value is not-null but is the unix epoch? this iteration would be unable to represent that case



##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var testDate0 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:30", -8*60*60 -30*60)
+var testDate1 = testDate0.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+11:00", +11*60*60)
+var testDate2 = testDate0.In(testZone2)
+
+func dict(index arrow.DataType) arrow.DataType {
+	return &arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered: false,
+	}
+}
+
+func ree(runEnds arrow.DataType) arrow.DataType {
+	v := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	v.ValueNullable = false
+	return v
+}
+
+// All tests use this in a for loop to make sure everything works for every possible
+// encoding of offsets (primitive, dictionary, run-end)
+var allAllowedOffsetTypes = []arrow.DataType{
+	// primitive offsetType
+	arrow.PrimitiveTypes.Int16,
+
+	// dict-encoded offsetType
+	dict(arrow.PrimitiveTypes.Uint8),
+	dict(arrow.PrimitiveTypes.Uint16),
+	dict(arrow.PrimitiveTypes.Uint32),
+	dict(arrow.PrimitiveTypes.Uint64),
+	dict(arrow.PrimitiveTypes.Int8),
+	dict(arrow.PrimitiveTypes.Int16),
+	dict(arrow.PrimitiveTypes.Int32),
+	dict(arrow.PrimitiveTypes.Int64),
+
+	// run-end encoded offsetType
+	ree(arrow.PrimitiveTypes.Int16),
+	ree(arrow.PrimitiveTypes.Int32),
+	ree(arrow.PrimitiveTypes.Int64),
+}
+
+func TestTimestampWithOffsetTypePrimitiveBasics(t *testing.T) {
+	typ := extensions.NewTimestampWithOffsetType(testTimeUnit)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     arrow.PrimitiveTypes.Int16,
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func assertDictBasics[I extensions.DictIndexType](t *testing.T, indexType I) {
+	typ := extensions.NewTimestampWithOffsetTypeDictionaryEncoded(testTimeUnit, indexType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name: "offset_minutes",
+				Type: dict(arrow.DataType(indexType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeDictionaryEncodedBasics(t *testing.T) {
+	assertDictBasics(t, &arrow.Uint8Type{})
+	assertDictBasics(t, &arrow.Uint16Type{})
+	assertDictBasics(t, &arrow.Uint32Type{})
+	assertDictBasics(t, &arrow.Uint64Type{})
+	assertDictBasics(t, &arrow.Int8Type{})
+	assertDictBasics(t, &arrow.Int16Type{})
+	assertDictBasics(t, &arrow.Int32Type{})
+	assertDictBasics(t, &arrow.Int64Type{})
+}
+
+func assertReeBasics[E extensions.TimestampWithOffsetRunEndsType](t *testing.T, runEndsType E) {
+	typ := extensions.NewTimestampWithOffsetTypeRunEndEncoded(testTimeUnit, runEndsType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name: "offset_minutes",
+				Type: ree(arrow.DataType(runEndsType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeRunEndEncodedBasics(t *testing.T) {
+	assertReeBasics(t, &arrow.Int16Type{})
+	assertReeBasics(t, &arrow.Int32Type{})
+	assertReeBasics(t, &arrow.Int64Type{})
+}
+
+func TestTimestampWithOffsetEquals(t *testing.T) {
+	// Completely different types are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewBool8Type()))
+
+	// Different time units are not equal
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	//
+	// // Different underlying storage type is not equal
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	// assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	//
+	// // Dict-encoding key type is not equal

Review Comment:
   why are these all commented out?



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   can you give me an example where it messes up the values? 
   
   Just to verify, the goal here is to take `t` and provide the same *instant of time* in UTC while figuring out what the offset was in minutes for the timezone, yes? i.e. given `2009-11-10 15:00:00 -0800 PST` this should return `2009-11-10 23:00:00 +0000 UTC` and `480`, correct?



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E TimestampWithOffsetRunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as TimestampWithOffsetRunEndsType is always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// If the timestamp is null, the returned time will be the unix epoch.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq[time.Time] {
+	return func(yield func(time.Time) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			offsetPhysicalIdx := 0
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				ts := time.Unix(0, 0)
+				if a.IsValid(i) {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				ts := time.Unix(0, 0)

Review Comment:
   this should be equivalent to just doing `var ts time.Time`, except faster since it avoids the extra processing of calling the `Unix` function.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated, but while fixing this I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap()` from the `Builder` interface. The default implementation does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state where the parent has greater length than the index/run-ends builder. This causes a panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? Also not sure if my change is good enough since it's very easy to forget to override `UnsafeAppendBoolToBitmap()`.
   
    https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated: I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap()` from the `Builder` interface. The default implementation does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state where the parent has greater length than the index/run-ends builder. This causes a panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? Also not sure if my change is good enough since it's very easy to forget to override `UnsafeAppendBoolToBitmap()`.
   
    https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated: I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap` from the `Builder` interface. This does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state where the parent has greater length than the index builder. This causes a panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "Copilot (via GitHub)" <gi...@apache.org>.
Copilot commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3560636846


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}

Review Comment:
   `timeFromFieldValues` derives the sign for the formatted zone name from `hours := offsetMinutes / 60`. For offsets with magnitude < 60 minutes (e.g. -30), `hours` becomes 0 and the resulting zone name becomes `UTC+00:30` even though the offset is negative. This produces incorrect string/JSON output for such offsets.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	_, offsetSeconds := t.Zone()
+	offsetMinutes := int16(offsetSeconds / 60)
+	ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
+	return ts, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Time{}
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// The second parameter indicates whether the timestamp is valid or not.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq2[time.Time, bool] {
+	return func(yield func(time.Time, bool) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			// Run-ends are absolute over the unsliced offsets child, so a sliced
+			// array must begin at the physical run covering its logical offset and
+			// advance using absolute positions (logicalOffset + i).
+			logicalOffset := reeOffsets.Offset()
+			offsetPhysicalIdx := reeOffsets.GetPhysicalOffset()
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if logicalOffset+i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	return slices.Collect(func(yield func(time.Time) bool) {
+		for time := range a.iterValues() {
+			if !yield(time) {
+				return
+			}
+		}
+	})
+}

Review Comment:
   In `Values()`, the loop variable is named `time`, which shadows the imported `time` package within the closure and makes the code harder to read.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,592 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeCustomOffset creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, fmt.Errorf("invalid offset type %s", offsetType)
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+type RunEndsType interface {
+	*arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type
+}
+
+// NewTimestampWithOffsetTypeDictionaryEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+// NewTimestampWithOffsetTypeRunEndEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded[E RunEndsType](unit arrow.TimeUnit, runEnds E) *TimestampWithOffsetType {
+	offsetType := arrow.RunEndEncodedOf(arrow.DataType(runEnds), arrow.PrimitiveTypes.Int16)
+
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, offsetType)
+	// SAFETY: This should never error as RunEndsType always a valid run ends type
+
+	return v
+
+}
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	return NewTimestampWithOffsetTypeCustomOffset(timeUnit, offsetType)
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName() &&
+		arrow.TypeEqual(b.StorageType(), other.StorageType())
+}
+
+func (b *TimestampWithOffsetType) OffsetType() arrow.DataType {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(1).Type
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Field(0).Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), b.OffsetType())
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	_, offsetSeconds := t.Zone()
+	offsetMinutes := int16(offsetSeconds / 60)
+	ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
+	return ts, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Time{}
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array returning the timestamp at each position.
+//
+// The second parameter indicates whether the timestamp is valid or not.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a *TimestampWithOffsetArray) iterValues() iter.Seq2[time.Time, bool] {
+	return func(yield func(time.Time, bool) bool) {
+		structs := a.Storage().(*array.Struct)
+		offsets := structs.Field(1)
+		if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+			timestampField := structs.Field(0)
+			timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+			timestamps := timestampField.(*array.Timestamp)
+
+			offsetValues := reeOffsets.Values().(*array.Int16)
+			// Run-ends are absolute over the unsliced offsets child, so a sliced
+			// array must begin at the physical run covering its logical offset and
+			// advance using absolute positions (logicalOffset + i).
+			logicalOffset := reeOffsets.Offset()
+			offsetPhysicalIdx := reeOffsets.GetPhysicalOffset()
+
+			var getRunEnd (func(int) int)
+			switch arr := reeOffsets.RunEndsArr().(type) {
+			case *array.Int16:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int32:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			case *array.Int64:
+				getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+			}
+
+			for i := 0; i < a.Len(); i++ {
+				if logicalOffset+i >= getRunEnd(offsetPhysicalIdx) {
+					offsetPhysicalIdx += 1
+				}
+
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp := timestamps.Value(i)
+					offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		} else {
+			for i := 0; i < a.Len(); i++ {
+				var ts time.Time
+				valid := a.IsValid(i)
+				if valid {
+					utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+					v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+					ts = v
+				}
+
+				if !yield(ts, valid) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	return slices.Collect(func(yield func(time.Time) bool) {
+		for time := range a.iterValues() {
+			if !yield(time) {
+				return
+			}
+		}
+	})
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	i := 0
+	for ts, valid := range a.iterValues() {
+		if !valid {
+			values[i] = nil
+		} else {
+			values[i] = &ts
+		}
+		i += 1
+	}
+	return json.Marshal(values)
+}

Review Comment:
   `MarshalJSON` stores `&ts` (address of the loop variable) into the `values` slice. The `ts` variable is reused across iterations, so all entries end up pointing to the same memory and will serialize as the last timestamp value.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3066922971


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/231cb1ec4211cf65d8e96fa3c5beec1a94bfc940



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3066918730


##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var testDate0 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:30", -8*60*60 -30*60)
+var testDate1 = testDate0.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+11:00", +11*60*60)
+var testDate2 = testDate0.In(testZone2)
+
+func dict(index arrow.DataType) arrow.DataType {
+	return &arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered: false,
+	}
+}
+
+func ree(runEnds arrow.DataType) arrow.DataType {
+	v := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	v.ValueNullable = false
+	return v
+}
+
+// All tests use this in a for loop to make sure everything works for every possible
+// encoding of offsets (primitive, dictionary, run-end)
+var allAllowedOffsetTypes = []arrow.DataType{
+	// primitive offsetType
+	arrow.PrimitiveTypes.Int16,
+
+	// dict-encoded offsetType
+	dict(arrow.PrimitiveTypes.Uint8),
+	dict(arrow.PrimitiveTypes.Uint16),
+	dict(arrow.PrimitiveTypes.Uint32),
+	dict(arrow.PrimitiveTypes.Uint64),
+	dict(arrow.PrimitiveTypes.Int8),
+	dict(arrow.PrimitiveTypes.Int16),
+	dict(arrow.PrimitiveTypes.Int32),
+	dict(arrow.PrimitiveTypes.Int64),
+
+	// run-end encoded offsetType
+	ree(arrow.PrimitiveTypes.Int16),
+	ree(arrow.PrimitiveTypes.Int32),
+	ree(arrow.PrimitiveTypes.Int64),
+}
+
+func TestTimestampWithOffsetTypePrimitiveBasics(t *testing.T) {
+	typ := extensions.NewTimestampWithOffsetType(testTimeUnit)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     arrow.PrimitiveTypes.Int16,
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func assertDictBasics[I extensions.DictIndexType](t *testing.T, indexType I) {
+	typ := extensions.NewTimestampWithOffsetTypeDictionaryEncoded(testTimeUnit, indexType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name: "offset_minutes",
+				Type: dict(arrow.DataType(indexType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeDictionaryEncodedBasics(t *testing.T) {
+	assertDictBasics(t, &arrow.Uint8Type{})
+	assertDictBasics(t, &arrow.Uint16Type{})
+	assertDictBasics(t, &arrow.Uint32Type{})
+	assertDictBasics(t, &arrow.Uint64Type{})
+	assertDictBasics(t, &arrow.Int8Type{})
+	assertDictBasics(t, &arrow.Int16Type{})
+	assertDictBasics(t, &arrow.Int32Type{})
+	assertDictBasics(t, &arrow.Int64Type{})
+}
+
+func assertReeBasics[E extensions.TimestampWithOffsetRunEndsType](t *testing.T, runEndsType E) {
+	typ := extensions.NewTimestampWithOffsetTypeRunEndEncoded(testTimeUnit, runEndsType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name: "offset_minutes",
+				Type: ree(arrow.DataType(runEndsType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeRunEndEncodedBasics(t *testing.T) {
+	assertReeBasics(t, &arrow.Int16Type{})
+	assertReeBasics(t, &arrow.Int32Type{})
+	assertReeBasics(t, &arrow.Int64Type{})
+}
+
+func TestTimestampWithOffsetEquals(t *testing.T) {
+	// Completely different types are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewBool8Type()))
+
+	// Different time units are not equal
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	//
+	// // Different underlying storage type is not equal
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	// assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	// assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	//
+	// // Dict-encoding key type is not equal

Review Comment:
   I was dumb. Fixed https://github.com/apache/arrow-go/pull/558/changes/02498067c12dfd04265234e715f8c8da34d22fbc



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated: I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap` from the `Builder` interface. The default implementation does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state where the parent has greater length than the index builder. This causes a panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758757968


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Unrelated: I found a bug on REE and dict encoding builders, where they were inheriting the `UnsafeAppendBoolToBitmap` from the `Builder` interface. This does not bump the length of the inner `idxBuilder` or `runEndsBuilder`, which causes it to be in an inconsistent state and panic when instancing a record batch.
   
   I added a fix here, but I should probably cherry pick it and submit it in a separate PR? https://github.com/apache/arrow-go/pull/558/changes/eff57e650e1fd2ea25193f0f6c9acc37e7e31c1f



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2754336572


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {

Review Comment:
   Commit: https://github.com/apache/arrow-go/pull/558/changes/cbf8f654f5cae86d1fe7cf382cf1b0a992f6cd7a
   
   Ok, I fixed this but I'm still not sure I like the look of the API. Turns out `arrow.PrimitiveTypes.*` are all `DataTypes` and the compiler complains `DataType` is too broad to fit in the generic constraints.
   
   You need to manually use the type struct like so:
   
   `NewTimestampWithOffsetTypeDictEncoded(arrow.Second, &arrow.Int8Type{})`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746054571


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   This is not semantically equal (I tried, it messes up the values, but only sometimes for some reason?).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade merged PR #558:
URL: https://github.com/apache/arrow-go/pull/558


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2640601044


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}

Review Comment:
   We can simplify this with something like:
   
   ```go
       if ts, compat := t.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {        
           timeUnit = ts.TimeUnit()
       } else {
           return
       }
   
       maybeOffset := t.Field(1)
       offsetType = maybeOffset.Type
       ok = t.Field(0).Name == "timestamp" && !t.Field(0).Nullable &&
           maybeOffset.Name == "offset_minutes" && isOffsetTypeOk(offsetType) &&
           !maybeOffset.Nullable
       return
   ```
   



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {

Review Comment:
   my personal preference for naming would be the inverse of this. The default `NewTimestampWithOffsetType` should be this one, while the version that takes a custom offsetType should be `NewTimestampWithOffsetTypeCustomOffset` or something to that effect. The simplest/most common usage should have the shorter name.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   are we still equal even if the time unit and structure of the offset minutes is different?



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}

Review Comment:
   if we change `iterValues` to return an `iter.Seq[time.Time]` then this becomes just `return slices.Collect(a.iterValues())`



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   these are non-nullable I thought? We should be pushing default 0,0 in the case where `!valids[i]`



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil

Review Comment:
   at this rate, just do `return NewTimestampWithOffsetType(timeUnit, offsetType)` since the function signature has an error, there's no reason to explicitly ignore it even if we know it'll always be valid.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}

Review Comment:
   as a follow-up we might be able to improve this by initializing a `arrow.TypedArray[int16]` in the `TimestampWithOffsetArray`. That way this can just be simplified to `offsetMinutes := offsets.Value(i)`
   
   For dictionaries we have `NewDictWrapper[int16](offsets)` which will wrap the `*arrow.Dictionary` and give the interface we want, but there isn't a generic wrapper for RunEndEncoded arrays yet that does the same. We'd need to create that before using this simplification, hence why I think this would be a good follow-up rather than doing this now.
   
   Aside from simplifying this function, it would also allow us to avoid this switch on every index by just using an interface that provides `Value(int) int16`. 



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()

Review Comment:
   use `.Field(0)` instead of `.Fields()[0]` ?



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:

Review Comment:
   This is a perfect case that would make sense to use named returns:
   
   
   ```suggestion
   func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
       st, compat := storageType.(*arrow.StructType)
       if !compat || st.NumFields() != 2 {
           return
       }                    
   ```
   
   etc...



##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,354 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var testDate1 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:00", -8*60*60)
+var testDate2 = testDate1.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+06:00", +6*60*60)

Review Comment:
   make sure there's a test with a zone that isn't evenly divisible by 60, perhaps a zone with an offset of `5:30` or something to that effect.
   
   Also, make sure we're testing a zone with an offset greater than 10 hours



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {

Review Comment:
   See the previous comment about potentially leveraging generics for simplification



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {

Review Comment:
   Another idea that might be interesting would be to use generics to avoid the need for the error output.
   
   For example:
   
   ```go
   func NewTimestampWithOffsetTypeDictEncoded[T arrow.IntType | arrow.UintType](unit arrow.TimeUnit) *TimestampWithOffsetType {
       offsetType := &arrow.DictionaryType{
           IndexType: arrow.GetDataType[T](),
           ValueType: arrow.PrimitiveTypes.Int16,
       }
   
       ...
   ```
   
   This would be used like `NewTimestampWithOffsetTypeDictEncoded[int8](arrow.Second)`
   
   or maybe:
   
   ```go
   type IntegralType interface {
       *arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
       *arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
   }
   
   func NewTimestampWithOffsetTypeDictEncoded[T IntegralType](unit arrow.TimeUnit, index T) *TimestampWithOffsetType {
       offsetType := &arrow.DictionaryType{
           IndexType: index,
           ValueType: arrow.PrimitiveTypes.Int16,
       }
   
       ...
   ```
   
   This could be used like `NewTimestampWithOffsetTypeDictEncoded(arrow.Second, arrow.PrimitiveTypes.Int8)`
   
   Leveraging the generics might make the usage simpler potentially by compile time enforcement of the proper types. This is just an idea, curious what you think



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   This would probably be simpler?
   
   ```go
   _, offsetSeconds := t.Zone()
   offsetMinutes = offsetSeconds / 60
   
   ts, _ := arrow.TimestampFromTime(t.UTC(), unit)
   return ts, offsetMinutes
   ```
   



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)

Review Comment:
   ```suggestion
       return json.Marshal(a.Values())
   ```



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {

Review Comment:
   instead, can we just use `iter.Seq`?



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)
+			}
+		}
+	case *array.Int16DictionaryBuilder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.Append(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   same as above, these are non-nullable according to the spec.



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)
+}
+
+func (a *TimestampWithOffsetArray) GetOneForMarshal(i int) interface{} {
+	if a.IsNull(i) {
+		return nil
+	}
+	return a.Value(i)
+}
+
+// TimestampWithOffsetBuilder is a convenience builder for the TimestampWithOffset extension type,
+// allowing arrays to be built with boolean values rather than the underlying storage type.
+type TimestampWithOffsetBuilder struct {
+	*array.ExtensionBuilder
+
+	// The layout used to parse any timestamps from strings. Defaults to time.RFC3339
+	Layout     string
+	unit       arrow.TimeUnit
+	offsetType arrow.DataType
+	// lastOffset is only used to determine when to start new runs with run-end encoded offsets
+	lastOffset int16
+}
+
+// NewTimestampWithOffsetBuilder creates a new TimestampWithOffsetBuilder, exposing a convenient and efficient interface
+// for writing time.Time values to the underlying storage array.
+func NewTimestampWithOffsetBuilder(mem memory.Allocator, unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetBuilder, error) {
+	dataType, err := NewTimestampWithOffsetType(unit, offsetType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimestampWithOffsetBuilder{
+		unit:             unit,
+		offsetType: 	  offsetType,
+		lastOffset:       math.MaxInt16,
+		Layout:           time.RFC3339,
+		ExtensionBuilder: array.NewExtensionBuilder(mem, dataType),
+	}, nil
+}
+
+func (b *TimestampWithOffsetBuilder) Append(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).Append(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.Append(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+
+}
+
+func (b *TimestampWithOffsetBuilder) UnsafeAppend(v time.Time) {
+	timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+	offsetMinutes16 := int16(offsetMinutes)
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+
+	structBuilder.Append(true)
+	structBuilder.FieldBuilder(0).(*array.TimestampBuilder).UnsafeAppend(timestamp)
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		offsets.UnsafeAppend(offsetMinutes16)
+	case *array.Int16DictionaryBuilder:
+		offsets.Append(offsetMinutes16)
+	case *array.RunEndEncodedBuilder:
+		if offsetMinutes != b.lastOffset {
+			offsets.Append(1)
+			offsets.ValueBuilder().(*array.Int16Builder).Append(offsetMinutes16)
+		} else {
+			offsets.ContinueRun(1)
+		}
+
+		b.lastOffset = offsetMinutes16
+	}
+}
+
+// By default, this will try to parse the string using the RFC3339 layout.
+//
+// You can change the default layout by using builder.SetLayout()
+func (b *TimestampWithOffsetBuilder) AppendValueFromString(s string) error {
+	if s == array.NullValueStr {
+		b.AppendNull()
+		return nil
+	}
+
+	parsed, err := time.Parse(b.Layout, s)
+	if err != nil {
+		return err
+	}
+
+	b.Append(parsed)
+	return nil
+}
+
+func (b *TimestampWithOffsetBuilder) AppendValues(values []time.Time, valids []bool) {
+	structBuilder := b.ExtensionBuilder.Builder.(*array.StructBuilder)
+	timestamps := structBuilder.FieldBuilder(0).(*array.TimestampBuilder)
+
+	structBuilder.AppendValues(valids)
+	// SAFETY: by this point we know all buffers have available space given the earlier
+	// call to structBuilder.AppendValues which calls Reserve internally, so it's OK to
+	// call UnsafeAppend on inner builders
+
+	switch offsets := structBuilder.FieldBuilder(1).(type) {
+	case *array.Int16Builder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.UnsafeAppend(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)
+			}
+		}
+	case *array.Int16DictionaryBuilder:
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsets.Append(offsetMinutes)
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)
+			}
+		}
+	case *array.RunEndEncodedBuilder:
+		offsetValuesBuilder := offsets.ValueBuilder().(*array.Int16Builder)
+		for i, v := range values {
+			timestamp, offsetMinutes := fieldValuesFromTime(v, b.unit)
+			if valids[i] {
+				timestamps.UnsafeAppend(timestamp)
+				offsetMinutes16 := int16(offsetMinutes)
+				if offsetMinutes != b.lastOffset {
+					offsets.Append(1)
+					offsetValuesBuilder.Append(offsetMinutes16)
+				} else {
+					offsets.ContinueRun(1)
+				}
+				b.lastOffset = offsetMinutes16
+			} else {
+				timestamps.UnsafeAppendBoolToBitmap(false)
+				offsets.UnsafeAppendBoolToBitmap(false)

Review Comment:
   same as above. these are non-nullable according to the spec



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2754336572


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {

Review Comment:
   Commit: https://github.com/apache/arrow-go/pull/558/changes/cbf8f654f5cae86d1fe7cf382cf1b0a992f6cd7a
   
   Ok, I fixed this but I'm still not sure I like the look of the API. Turns out `arrow.PrimitiveTypes.*` are all `DataTypes` and the compiler complains `DataType` is too broad to fit in the generic constraints.
   
   You need to manually use the type struct like so:
   
   `NewTimestampWithOffsetTypeDictEncoded(arrow.Second, &arrow.Int8Type{})`
   
   The other disadvantage is that I can't just iterate over a list of `DataType` like I was doing in tests before, you need to specify the specialization of the function at compile-time. Which might not be a problem for users, but wanted to point that out...



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746054571


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   This is not semantically equal (I tried, it messes up the values, only sometimes?).
   
   What I'm trying to accomplish here is get the `naiveUtc` by converting all the "raw" values without actually going through Go's timezone conversion logic, i.e simply dropping the timezone without any conversion of the underlying values. Then I apply my own timezone based on the offset.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746042227


##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,354 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var testDate1 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:00", -8*60*60)
+var testDate2 = testDate1.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+06:00", +6*60*60)

Review Comment:
   Fixed here: https://github.com/apache/arrow-go/pull/558/changes/cee17a3e4e4677124c07cd71359f724f4fce5d50



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2758752868


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/2762d64b50221ac82ed0a22231eb8478427823ed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3066921124


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/76e3ace8d45e9162706282203eff81766db8dd97



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/76e3ace8d45e9162706282203eff81766db8dd97



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085996363


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())

Review Comment:
   Actually I'm a dumdum. Fixed: https://github.com/apache/arrow-go/pull/558/changes/3a9a8f2af3bb37058418a2ebb94ae0bdd4e7ed5e



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3067134782


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}

Review Comment:
   Did this in a separate PR: https://github.com/apache/arrow-go/pull/758



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3066923977


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/f42417b775645a0fd4b4ec2502acb1a327cc9222



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085794270


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}
+
+func (a *TimestampWithOffsetArray) ValueStr(i int) string {
+	switch {
+	case a.IsNull(i):
+		return array.NullValueStr
+	default:
+		return a.Value(i).String()
+	}
+}
+
+func (a *TimestampWithOffsetArray) MarshalJSON() ([]byte, error) {
+	values := make([]interface{}, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		values[i] = timestamp
+	})
+	return json.Marshal(values)

Review Comment:
   https://github.com/apache/arrow-go/pull/558/changes/d955cdb46fe2d17c547e51f9f5e123b2edcb7d76
   
   Fix here.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3085514700


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,577 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"iter"
+	"math"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) &&
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+		// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+		// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+		// stream
+		// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (unit arrow.TimeUnit, offsetType arrow.DataType, ok bool) {
+	unit = arrow.Second
+	offsetType = arrow.PrimitiveTypes.Int16
+	ok = false
+
+	st, compat := storageType.(*arrow.StructType)
+	if !compat || st.NumFields() != 2 {
+		return
+	}
+
+	if ts, compat := st.Field(0).Type.(*arrow.TimestampType); compat && ts.TimeZone == "UTC" {
+		unit = ts.TimeUnit()
+	} else {
+		return
+	}
+
+	maybeOffset := st.Field(1)
+	offsetType = maybeOffset.Type
+
+	ok = st.Field(0).Name == "timestamp" &&
+		!st.Field(0).Nullable &&
+		maybeOffset.Name == "offset_minutes" &&
+		isOffsetTypeOk(offsetType) &&
+		!maybeOffset.Nullable
+	return
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetTypeCustomOffset(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+type DictIndexType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded[I DictIndexType](unit arrow.TimeUnit, index I) *TimestampWithOffsetType {
+	offsetType := arrow.DictionaryType{
+		IndexType: arrow.DataType(index),
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	v, _ := NewTimestampWithOffsetTypeCustomOffset(unit, &offsetType)
+	// SAFETY: This should never error as DictIndexType is always a valid index type
+
+	return v
+}
+
+type TimestampWithOffsetRunEndsType interface {
+	*arrow.Int8Type | *arrow.Int16Type | *arrow.Int32Type | *arrow.Int64Type |
+		*arrow.Uint8Type | *arrow.Uint16Type | *arrow.Uint32Type | *arrow.Uint64Type

Review Comment:
   Actually I had to revert. Spec says those types are different:
   
   Dict index is signed int of 8-64 bits
   > The index type of a Dictionary type can only be an integer type, preferably signed, with width 8 to 64 bits.
   
   Run ends is signed int of 16-64 bits
   > The run end type of a Run-End Encoded type can only be a signed integer type with width 16 to 64 bits.
   
   https://arrow.apache.org/docs/format/Columnar.html#data-types



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r2746058049


##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



##########
arrow/extensions/timestamp_with_offset.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+)
+
+// TimestampWithOffsetType represents a timestamp column that stores a timezone offset per row instead of
+// applying the same timezone offset to the entire column.
+type TimestampWithOffsetType struct {
+	arrow.ExtensionBase
+}
+
+func isOffsetTypeOk(offsetType arrow.DataType) bool {
+	switch offsetType := offsetType.(type) {
+	case *arrow.Int16Type:
+		return true
+	case *arrow.DictionaryType:
+		return arrow.IsInteger(offsetType.IndexType.ID()) && arrow.TypeEqual(offsetType.ValueType, arrow.PrimitiveTypes.Int16)
+	case *arrow.RunEndEncodedType:
+		return offsetType.ValidRunEndsType(offsetType.RunEnds()) && 
+			arrow.TypeEqual(offsetType.Encoded(), arrow.PrimitiveTypes.Int16)
+			// FIXME: Technically this should be non-nullable, but a Arrow IPC does not deserialize
+			// ValueNullable properly, so enforcing this here would always fail when reading from an IPC
+			// stream
+			// !offsetType.ValueNullable
+	default:
+		return false
+	}
+}
+
+// Whether the storageType is compatible with TimestampWithOffset.
+//
+// Returns (time_unit, offset_type, ok). If ok is false, time_unit and offset_type are garbage.
+func isDataTypeCompatible(storageType arrow.DataType) (arrow.TimeUnit, arrow.DataType, bool) {
+	timeUnit := arrow.Second
+	offsetType := arrow.PrimitiveTypes.Int16
+	switch t := storageType.(type) {
+	case *arrow.StructType:
+		if t.NumFields() != 2 {
+			return timeUnit, offsetType, false
+		}
+
+		maybeTimestamp := t.Field(0)
+		maybeOffset := t.Field(1)
+
+		timestampOk := false
+		switch timestampType := maybeTimestamp.Type.(type) {
+		case *arrow.TimestampType:
+			if timestampType.TimeZone == "UTC" {
+				timestampOk = true
+				timeUnit = timestampType.TimeUnit()
+			}
+		default:
+		}
+
+		offsetOk := isOffsetTypeOk(maybeOffset.Type)
+
+		ok := maybeTimestamp.Name == "timestamp" &&
+			timestampOk &&
+			!maybeTimestamp.Nullable &&
+			maybeOffset.Name == "offset_minutes" &&
+			offsetOk &&
+			!maybeOffset.Nullable
+
+		return timeUnit, maybeOffset.Type, ok
+	default:
+		return timeUnit, offsetType, false
+	}
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=O), where T is any TimeUnit and O is a valid offset type.
+//
+// The error will be populated if the data type is not a valid encoding of the offsets field.
+func NewTimestampWithOffsetType(unit arrow.TimeUnit, offsetType arrow.DataType) (*TimestampWithOffsetType, error) {
+	if !isOffsetTypeOk(offsetType) {
+		return nil, errors.New(fmt.Sprintf("Invalid offset type %s", offsetType))
+	}
+
+	return &TimestampWithOffsetType{
+		ExtensionBase: arrow.ExtensionBase{
+			Storage: arrow.StructOf(
+				arrow.Field{
+					Name: "timestamp",
+					Type: &arrow.TimestampType{
+						Unit:     unit,
+						TimeZone: "UTC",
+					},
+					Nullable: false,
+				},
+				arrow.Field{
+					Name:     "offset_minutes",
+					Type:     offsetType,
+					Nullable: false,
+				},
+			),
+		},
+	}, nil
+}
+
+
+// NewTimestampWithOffsetTypePrimitiveEncoded creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Int16), where T is any TimeUnit.
+func NewTimestampWithOffsetTypePrimitiveEncoded(unit arrow.TimeUnit) *TimestampWithOffsetType {
+	v, _ := NewTimestampWithOffsetType(unit, arrow.PrimitiveTypes.Int16)
+	// SAFETY: This should never error as Int16 is always a valid offset type
+
+	return v
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=Dictionary(I, Int16)), where T is any TimeUnit and I is a
+// valid Dictionary index type.
+//
+// The error will be populated if the index is not a valid dictionary-encoding index type.
+func NewTimestampWithOffsetTypeDictionaryEncoded(unit arrow.TimeUnit, index arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+	return NewTimestampWithOffsetType(unit, &offsetType)
+}
+
+// NewTimestampWithOffsetType creates a new TimestampWithOffsetType with the underlying storage type set correctly to
+// Struct(timestamp=Timestamp(T, "UTC"), offset_minutes=RunEndEncoded(E, Int16)), where T is any TimeUnit and E is a
+// valid run-ends type.
+//
+// The error will be populated if runEnds is not a valid run-end encoding run-ends type.
+func NewTimestampWithOffsetTypeRunEndEncoded(unit arrow.TimeUnit, runEnds arrow.DataType) (*TimestampWithOffsetType, error) {
+	offsetType := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	if !offsetType.ValidRunEndsType(runEnds) {
+		return nil, errors.New(fmt.Sprintf("Invalid run-ends type %s", runEnds))
+	}
+
+	return NewTimestampWithOffsetType(unit, offsetType)
+}
+
+
+func (b *TimestampWithOffsetType) ArrayType() reflect.Type {
+	return reflect.TypeOf(TimestampWithOffsetArray{})
+}
+
+func (b *TimestampWithOffsetType) ExtensionName() string { return "arrow.timestamp_with_offset" }
+
+func (b *TimestampWithOffsetType) String() string {
+	return fmt.Sprintf("extension<%s>", b.ExtensionName())
+}
+
+func (e *TimestampWithOffsetType) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil
+}
+
+func (b *TimestampWithOffsetType) Serialize() string { return "" }
+
+func (b *TimestampWithOffsetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) {
+	timeUnit, offsetType, ok := isDataTypeCompatible(storageType)
+	if !ok {
+		return nil, fmt.Errorf("invalid storage type for TimestampWithOffsetType: %s", storageType.Name())
+	}
+
+	v, _ := NewTimestampWithOffsetType(timeUnit, offsetType)
+	// SAFETY: the offsetType has already been checked by isDataTypeCompatible, so we can ignore the error
+
+	return v, nil
+}
+
+func (b *TimestampWithOffsetType) ExtensionEquals(other arrow.ExtensionType) bool {
+	return b.ExtensionName() == other.ExtensionName()
+}
+
+func (b *TimestampWithOffsetType) TimeUnit() arrow.TimeUnit {
+	return b.ExtensionBase.Storage.(*arrow.StructType).Fields()[0].Type.(*arrow.TimestampType).TimeUnit()
+}
+
+func (b *TimestampWithOffsetType) NewBuilder(mem memory.Allocator) array.Builder {
+	v, _ := NewTimestampWithOffsetBuilder(mem, b.TimeUnit(), arrow.PrimitiveTypes.Int16)
+	// SAFETY: This will never error as Int16 is always a valid type for the offset field
+
+	return v
+}
+
+// TimestampWithOffsetArray is a simple array of struct
+type TimestampWithOffsetArray struct {
+	array.ExtensionArrayBase
+}
+
+func (a *TimestampWithOffsetArray) String() string {
+	var o strings.Builder
+	o.WriteString("[")
+	for i := 0; i < a.Len(); i++ {
+		if i > 0 {
+			o.WriteString(" ")
+		}
+		switch {
+		case a.IsNull(i):
+			o.WriteString(array.NullValueStr)
+		default:
+			fmt.Fprintf(&o, "\"%s\"", a.Value(i))
+		}
+	}
+	o.WriteString("]")
+	return o.String()
+}
+
+func timeFromFieldValues(utcTimestamp arrow.Timestamp, offsetMinutes int16, unit arrow.TimeUnit) time.Time {
+	hours := offsetMinutes / 60
+	minutes := offsetMinutes % 60
+	if minutes < 0 {
+		minutes = -minutes
+	}
+
+	loc := time.FixedZone(fmt.Sprintf("UTC%+03d:%02d", hours, minutes), int(offsetMinutes)*60)
+	return utcTimestamp.ToTime(unit).In(loc)
+}
+
+func fieldValuesFromTime(t time.Time, unit arrow.TimeUnit) (arrow.Timestamp, int16) {
+	// naive "bitwise" conversion to UTC, keeping the underlying date the same
+	utc := t.UTC()
+	naiveUtc := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.UTC)
+	offsetMinutes := int16(naiveUtc.Sub(t).Minutes())
+	// SAFETY: unit MUST have been validated to a valid arrow.TimeUnit value before
+	// this function. Otherwise, ignoring this error is not safe.
+	timestamp, _ := arrow.TimestampFromTime(utc, unit)
+	return timestamp, offsetMinutes
+}
+
+// Get the raw arrow values at the given index
+//
+// SAFETY: the value at i must not be nil
+func (a *TimestampWithOffsetArray) rawValueUnsafe(i int) (arrow.Timestamp, int16, arrow.TimeUnit) {
+	structs := a.Storage().(*array.Struct)
+
+	timestampField := structs.Field(0)
+	timestamps := timestampField.(*array.Timestamp)
+
+	timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+	utcTimestamp := timestamps.Value(i)
+
+	var offsetMinutes int16
+
+	switch offsets := structs.Field(1).(type) {
+	case *array.Int16:
+		offsetMinutes = offsets.Value(i)
+	case *array.Dictionary:
+		offsetMinutes = offsets.Dictionary().(*array.Int16).Value(offsets.GetValueIndex(i))
+	case *array.RunEndEncoded:
+		offsetMinutes = offsets.Values().(*array.Int16).Value(offsets.GetPhysicalIndex(i))
+	}
+
+	return utcTimestamp, offsetMinutes, timeUnit
+}
+
+func (a *TimestampWithOffsetArray) Value(i int) time.Time {
+	if a.IsNull(i) {
+		return time.Unix(0, 0)
+	}
+	utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+	return timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+}
+
+// Iterates over the array and calls the callback with the timestamp at each position. If it's null,
+// the timestamp will be nil.
+//
+// This will iterate using the fastest method given the underlying storage array
+func (a* TimestampWithOffsetArray) iterValues(callback func(i int, utcTimestamp *time.Time)) {
+	structs := a.Storage().(*array.Struct)
+	offsets := structs.Field(1)
+	if reeOffsets, isRee := offsets.(*array.RunEndEncoded); isRee {
+		timestampField := structs.Field(0)
+		timeUnit := timestampField.DataType().(*arrow.TimestampType).Unit
+		timestamps := timestampField.(*array.Timestamp)
+
+		offsetValues := reeOffsets.Values().(*array.Int16)
+		offsetPhysicalIdx := 0
+
+		var getRunEnd (func(int) int)
+		switch arr := reeOffsets.RunEndsArr().(type) {
+		case *array.Int16:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int32:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		case *array.Int64:
+			getRunEnd = func(idx int) int { return int(arr.Value(idx)) }
+		}
+
+		for i := 0; i < a.Len(); i++ {
+			if i >= getRunEnd(offsetPhysicalIdx) {
+				offsetPhysicalIdx += 1
+			}
+
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp := timestamps.Value(i)
+				offsetMinutes := offsetValues.Value(offsetPhysicalIdx)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	} else {
+		for i := 0; i < a.Len(); i++ {
+			timestamp := (*time.Time)(nil)
+			if a.IsValid(i) {
+				utcTimestamp, offsetMinutes, timeUnit := a.rawValueUnsafe(i)
+				v := timeFromFieldValues(utcTimestamp, offsetMinutes, timeUnit)
+				timestamp = &v
+			} 
+
+			callback(i, timestamp)
+		}
+	}
+}
+
+
+func (a *TimestampWithOffsetArray) Values() []time.Time {
+	values := make([]time.Time, a.Len())
+	a.iterValues(func(i int, timestamp *time.Time) {
+		if timestamp == nil {
+			values[i] = time.Unix(0, 0)
+		} else {
+			values[i] = *timestamp
+		}
+	})
+	return values
+}

Review Comment:
   Fixed: https://github.com/apache/arrow-go/pull/558/changes/bd12287760927080eda309594b36dc9e2ec813ff



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3581375707


##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,591 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var epoch = time.Unix(0, 0).In(time.FixedZone("UTC+00:00", 0))
+
+var testDate0 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:30", -8*60*60-30*60)
+var testDate1 = testDate0.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+11:00", +11*60*60)
+var testDate2 = testDate0.In(testZone2)
+
+func dict(index arrow.DataType) arrow.DataType {
+	return &arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+}
+
+func ree(runEnds arrow.DataType) arrow.DataType {
+	v := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	v.ValueNullable = false
+	return v
+}
+
+// All tests use this in a for loop to make sure everything works for every possible
+// encoding of offsets (primitive, dictionary, run-end)
+var allAllowedOffsetTypes = make(map[string]arrow.DataType)
+
+func init() {
+	// primitive offsetType
+	allAllowedOffsetTypes["primitive-int16"] = arrow.PrimitiveTypes.Int16
+
+	// dict-encoded offsetType
+	allAllowedOffsetTypes["dict-Uint8"] = dict(arrow.PrimitiveTypes.Uint8)
+	allAllowedOffsetTypes["dict-Uint16"] = dict(arrow.PrimitiveTypes.Uint16)
+	allAllowedOffsetTypes["dict-Uint32"] = dict(arrow.PrimitiveTypes.Uint32)
+	allAllowedOffsetTypes["dict-Uint64"] = dict(arrow.PrimitiveTypes.Uint64)
+	allAllowedOffsetTypes["dict-Int8"] = dict(arrow.PrimitiveTypes.Int8)
+	allAllowedOffsetTypes["dict-Int16"] = dict(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["dict-Int32"] = dict(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["dict-Int64"] = dict(arrow.PrimitiveTypes.Int64)
+
+	// run-end encoded offsetType
+	allAllowedOffsetTypes["ree-Int16"] = ree(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["ree-Int32"] = ree(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["ree-Int64"] = ree(arrow.PrimitiveTypes.Int64)
+}
+
+func TestTimestampWithOffsetTypePrimitiveBasics(t *testing.T) {
+	typ := extensions.NewTimestampWithOffsetType(testTimeUnit)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     arrow.PrimitiveTypes.Int16,
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func assertDictBasics[I extensions.DictIndexType](t *testing.T, indexType I) {
+	typ := extensions.NewTimestampWithOffsetTypeDictionaryEncoded(testTimeUnit, indexType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     dict(arrow.DataType(indexType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeDictionaryEncodedBasics(t *testing.T) {
+	assertDictBasics(t, &arrow.Uint8Type{})
+	assertDictBasics(t, &arrow.Uint16Type{})
+	assertDictBasics(t, &arrow.Uint32Type{})
+	assertDictBasics(t, &arrow.Uint64Type{})
+	assertDictBasics(t, &arrow.Int8Type{})
+	assertDictBasics(t, &arrow.Int16Type{})
+	assertDictBasics(t, &arrow.Int32Type{})
+	assertDictBasics(t, &arrow.Int64Type{})
+}
+
+func assertReeBasics[E extensions.RunEndsType](t *testing.T, runEndsType E) {
+	typ := extensions.NewTimestampWithOffsetTypeRunEndEncoded(testTimeUnit, runEndsType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     ree(arrow.DataType(runEndsType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeRunEndEncodedBasics(t *testing.T) {
+	assertReeBasics(t, &arrow.Int16Type{})
+	assertReeBasics(t, &arrow.Int32Type{})
+	assertReeBasics(t, &arrow.Int64Type{})
+}
+
+func TestTimestampWithOffsetEquals(t *testing.T) {
+	// Completely different types are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewBool8Type()))
+
+	// Different time units are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+
+	// Different underlying storage type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+
+	// Dict-encoding key type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+
+	// REE index type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+
+	// Equals OK
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Nanosecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+}
+
+func TestTimestampWithOffsetExtensionBuilder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, _ := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+
+			builder.Append(testDate0)
+			builder.AppendNull()
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			builder.Append(epoch)
+
+			// it should build the array with the correct size
+			arr := builder.NewArray()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+			assert.Equal(t, 5, arr.Data().Len())
+			defer arr.Release()
+
+			// typedArr.Value(i) should return values adjusted for their original timezone
+			assert.Equal(t, testDate0, typedArr.Value(0))
+			assert.Equal(t, testDate1, typedArr.Value(2))
+			assert.Equal(t, testDate2, typedArr.Value(3))
+			assert.Equal(t, epoch, typedArr.Value(4))
+
+			// storage TimeUnit should be the same as we pass in to the builder, and storage timezone should be UTC
+			timestampStructField := typedArr.Storage().(*array.Struct).Field(0)
+			timestampStructDataType := timestampStructField.DataType().(*arrow.TimestampType)
+			assert.Equal(t, timestampStructDataType.Unit, testTimeUnit)
+			assert.Equal(t, timestampStructDataType.TimeZone, "UTC")
+
+			// stored values should be equivalent to the raw values in UTC
+			timestampsArr := timestampStructField.(*array.Timestamp)
+			assert.Equal(t, testDate0.In(time.UTC), timestampsArr.Value(0).ToTime(testTimeUnit))
+			assert.Equal(t, testDate1.In(time.UTC), timestampsArr.Value(2).ToTime(testTimeUnit))
+			assert.Equal(t, testDate2.In(time.UTC), timestampsArr.Value(3).ToTime(testTimeUnit))
+			assert.Equal(t, epoch.In(time.UTC), timestampsArr.Value(4).ToTime(testTimeUnit))
+
+			// the array should encode itself as JSON and string
+			arrStr := arr.String()
+			assert.Equal(t, fmt.Sprintf(`["%[1]s" (null) "%[2]s" "%[3]s" "%[4]s"]`, testDate0, testDate1, testDate2, epoch), arrStr)
+			jsonStr, err := json.Marshal(arr)
+			assert.NoError(t, err)
+
+			// roundtripping from JSON with array.FromJSON should work
+			expectedDataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			roundtripped, _, err := array.FromJSON(mem, expectedDataType, bytes.NewReader(jsonStr))
+			defer roundtripped.Release()
+			assert.NoError(t, err)
+			assert.Truef(t, array.Equal(arr, roundtripped), "expected %s\n\ngot %s", arr, roundtripped)
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesLeadingNull(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			// A leading null as the very first operation on a fresh builder must not
+			// corrupt the offsets. Reading every value below would panic on a
+			// run-end encoded array whose run-ends and values children fell out of
+			// sync (which happened when a run was continued before one existed).
+			builder.AppendValues([]time.Time{epoch, testDate1, testDate2}, []bool{false, true, true})
+
+			arr := builder.NewArray()
+			defer arr.Release()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+			require.Equal(t, 3, arr.Data().Len())
+			assert.True(t, typedArr.IsNull(0))
+			assert.Equal(t, testDate1, typedArr.Value(1))
+			assert.Equal(t, testDate2, typedArr.Value(2))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderRunEndEncodedNullContinuesRun(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// The same offset surrounds a null. The null must continue the current run
+	// rather than reset the tracked offset, otherwise one logical run is split
+	// into two adjacent runs that share the same value.
+	builder.AppendValues([]time.Time{testDate1, epoch, testDate1}, []bool{true, false, true})
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	offsets := typedArr.Storage().(*array.Struct).Field(1).(*array.RunEndEncoded)
+	assert.Equal(t, 1, offsets.Values().Len())
+
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesNilValids(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// nil and empty validity slices must both be treated as all-valid so the
+	// parent struct and its child builders stay the same length.
+	validsVariants := map[string][]bool{"nil": nil, "empty": {}}
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		for validsName, valids := range validsVariants {
+			t.Run(name+"/"+validsName, func(t *testing.T) {
+				builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+				require.NoError(t, err)
+
+				builder.AppendValues([]time.Time{testDate0, testDate1, testDate2}, valids)
+
+				arr := builder.NewArray()
+				defer arr.Release()
+				typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+				require.Equal(t, 3, arr.Data().Len())
+				assert.Equal(t, testDate0, typedArr.Value(0))
+				assert.Equal(t, testDate1, typedArr.Value(1))
+				assert.Equal(t, testDate2, typedArr.Value(2))
+			})
+		}
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesMismatchedValidsPanics(t *testing.T) {
+	builder, err := extensions.NewTimestampWithOffsetBuilder(memory.DefaultAllocator, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	defer builder.Release()
+
+	assert.Panics(t, func() {
+		builder.AppendValues([]time.Time{testDate0, testDate1}, []bool{true})
+	})
+}
+
+func TestTimestampWithOffsetBuilderReuseAfterNewArray(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			builder.Append(testDate1)
+			arr1 := builder.NewArray()
+			defer arr1.Release()
+
+			// Reusing the builder must start a fresh run even when the first value
+			// repeats the previous array's offset; otherwise a run-end-encoded
+			// offset is continued on a freshly reset (empty) builder.
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			arr2 := builder.NewArray()
+			defer arr2.Release()
+
+			typedArr := arr2.(*extensions.TimestampWithOffsetArray)
+			require.Equal(t, 2, arr2.Data().Len())
+			assert.Equal(t, testDate1, typedArr.Value(0))
+			assert.Equal(t, testDate2, typedArr.Value(1))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendNullBetweenEqualOffsets(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// AppendNull appends a null offset run; the following value repeats the
+	// pre-null offset and must start its own run instead of being folded into
+	// the null run (which would decode the trailing value's offset as null).
+	builder.Append(testDate1)
+	builder.AppendNull()
+	builder.Append(testDate1)
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	require.Equal(t, 3, arr.Data().Len())
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetRunEndEncodedSliced(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for _, runEnds := range []arrow.DataType{
+		arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64,
+	} {
+		t.Run(runEnds.String(), func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(runEnds))
+			require.NoError(t, err)
+
+			// Offsets [0,0,-510,-510,-510,660,0] give run-ends [2,5,6,7], so a
+			// slice starting at index 3 begins inside a later run: the case where
+			// an offset-unaware physical index decodes the wrong run.
+			values := []time.Time{testDate0, testDate0, testDate1, testDate1, testDate1, testDate2, epoch}
+			builder.AppendValues(values, nil)
+			arr := builder.NewArray().(*extensions.TimestampWithOffsetArray)
+			defer arr.Release()
+
+			want := arr.Values()
+
+			for start := 0; start < arr.Len(); start++ {
+				for end := start; end <= arr.Len(); end++ {
+					sliced := array.NewSlice(arr, int64(start), int64(end)).(*extensions.TimestampWithOffsetArray)
+					got := sliced.Values()
+					require.Equalf(t, end-start, len(got), "slice [%d:%d]", start, end)
+					for k := 0; k < end-start; k++ {
+						require.Truef(t, want[start+k].Equal(got[k]), "slice [%d:%d] row %d instant: want %s got %s", start, end, k, want[start+k], got[k])
+						_, wantOff := want[start+k].Zone()
+						_, gotOff := got[k].Zone()
+						require.Equalf(t, wantOff, gotOff, "slice [%d:%d] row %d offset", start, end, k)
+					}
+					sliced.Release()
+				}
+			}
+		})
+	}
+}
+
+func TestTimestampWithOffsetSubHourNegativeZone(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// A negative offset smaller than one hour must keep its sign in the zone
+	// name (regression: -30 minutes formatted as "UTC+00:30" instead of "UTC-00:30").
+	b, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	b.Append(testDate0.In(time.FixedZone("UTC-00:30", -30*60)))
+	arr := b.NewArray().(*extensions.TimestampWithOffsetArray)
+	defer arr.Release()
+
+	name, off := arr.Value(0).Zone()
+	require.Equal(t, -30*60, off)
+	require.Equal(t, "UTC-00:30", name)
+}
+
+func TestTimestampWithOffsetExtensionRecordBuilder(t *testing.T) {
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			dataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			schema := arrow.NewSchema([]arrow.Field{
+				{
+					Name:     "timestamp_with_offset",
+					Nullable: true,
+					Type:     dataType,
+				},
+			}, nil)
+			builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
+			defer builder.Release()
+
+			fieldBuilder := builder.Field(0).(*extensions.TimestampWithOffsetBuilder)
+
+			// append a simple time.Time
+			fieldBuilder.Append(testDate0)
+
+			// append the epoch
+			fieldBuilder.Append(epoch)
+
+			// append a null and 2 time.Time all at once
+			values := []time.Time{
+				time.Unix(0, 0).In(time.UTC),
+				testDate1,
+				testDate2,
+			}
+			valids := []bool{false, true, true}
+			fieldBuilder.AppendValues(values, valids)
+
+			// append a value from RFC3339 string
+			fieldBuilder.AppendValueFromString(testDate0.Format(time.RFC3339))
+
+			// append value formatted in a different string layout
+			fieldBuilder.Layout = time.RFC3339Nano
+			fieldBuilder.AppendValueFromString(testDate1.Format(time.RFC3339Nano))
+
+			record := builder.NewRecordBatch()
+
+			// Record batch should JSON-encode values containing per-row timezone info
+			json, err := record.MarshalJSON()
+			require.NoError(t, err)
+			expect := `[{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"1970-01-01T00:00:00Z"}
+,{"timestamp_with_offset":null}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+,{"timestamp_with_offset":"2025-01-01T11:00:00+11:00"}
+,{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+]`
+			require.Equal(t, expect, string(json))
+
+			// Record batch roundtrip to JSON should work
+			roundtripped, _, err := array.RecordFromJSON(memory.DefaultAllocator, schema, bytes.NewReader(json))
+			require.NoError(t, err)
+			defer roundtripped.Release()
+			require.Equal(t, schema, roundtripped.Schema())
+			// Avoid array.RecordEqual here: it compares the inner non-nullable

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Re: [PR] Add `TimestampWithOffset` canonical extension type [arrow-go]

Posted by "serramatutu (via GitHub)" <gi...@apache.org>.
serramatutu commented on code in PR #558:
URL: https://github.com/apache/arrow-go/pull/558#discussion_r3562127195


##########
arrow/extensions/timestamp_with_offset_test.go:
##########
@@ -0,0 +1,591 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensions_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/extensions"
+	"github.com/apache/arrow-go/v18/arrow/ipc"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+	"github.com/apache/arrow-go/v18/internal/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testTimeUnit = arrow.Microsecond
+
+var epoch = time.Unix(0, 0).In(time.FixedZone("UTC+00:00", 0))
+
+var testDate0 = time.Date(2025, 01, 01, 00, 00, 00, 00, time.FixedZone("UTC+00:00", 0))
+
+var testZone1 = time.FixedZone("UTC-08:30", -8*60*60-30*60)
+var testDate1 = testDate0.In(testZone1)
+
+var testZone2 = time.FixedZone("UTC+11:00", +11*60*60)
+var testDate2 = testDate0.In(testZone2)
+
+func dict(index arrow.DataType) arrow.DataType {
+	return &arrow.DictionaryType{
+		IndexType: index,
+		ValueType: arrow.PrimitiveTypes.Int16,
+		Ordered:   false,
+	}
+}
+
+func ree(runEnds arrow.DataType) arrow.DataType {
+	v := arrow.RunEndEncodedOf(runEnds, arrow.PrimitiveTypes.Int16)
+	v.ValueNullable = false
+	return v
+}
+
+// All tests use this in a for loop to make sure everything works for every possible
+// encoding of offsets (primitive, dictionary, run-end)
+var allAllowedOffsetTypes = make(map[string]arrow.DataType)
+
+func init() {
+	// primitive offsetType
+	allAllowedOffsetTypes["primitive-int16"] = arrow.PrimitiveTypes.Int16
+
+	// dict-encoded offsetType
+	allAllowedOffsetTypes["dict-Uint8"] = dict(arrow.PrimitiveTypes.Uint8)
+	allAllowedOffsetTypes["dict-Uint16"] = dict(arrow.PrimitiveTypes.Uint16)
+	allAllowedOffsetTypes["dict-Uint32"] = dict(arrow.PrimitiveTypes.Uint32)
+	allAllowedOffsetTypes["dict-Uint64"] = dict(arrow.PrimitiveTypes.Uint64)
+	allAllowedOffsetTypes["dict-Int8"] = dict(arrow.PrimitiveTypes.Int8)
+	allAllowedOffsetTypes["dict-Int16"] = dict(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["dict-Int32"] = dict(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["dict-Int64"] = dict(arrow.PrimitiveTypes.Int64)
+
+	// run-end encoded offsetType
+	allAllowedOffsetTypes["ree-Int16"] = ree(arrow.PrimitiveTypes.Int16)
+	allAllowedOffsetTypes["ree-Int32"] = ree(arrow.PrimitiveTypes.Int32)
+	allAllowedOffsetTypes["ree-Int64"] = ree(arrow.PrimitiveTypes.Int64)
+}
+
+func TestTimestampWithOffsetTypePrimitiveBasics(t *testing.T) {
+	typ := extensions.NewTimestampWithOffsetType(testTimeUnit)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     arrow.PrimitiveTypes.Int16,
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func assertDictBasics[I extensions.DictIndexType](t *testing.T, indexType I) {
+	typ := extensions.NewTimestampWithOffsetTypeDictionaryEncoded(testTimeUnit, indexType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     dict(arrow.DataType(indexType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeDictionaryEncodedBasics(t *testing.T) {
+	assertDictBasics(t, &arrow.Uint8Type{})
+	assertDictBasics(t, &arrow.Uint16Type{})
+	assertDictBasics(t, &arrow.Uint32Type{})
+	assertDictBasics(t, &arrow.Uint64Type{})
+	assertDictBasics(t, &arrow.Int8Type{})
+	assertDictBasics(t, &arrow.Int16Type{})
+	assertDictBasics(t, &arrow.Int32Type{})
+	assertDictBasics(t, &arrow.Int64Type{})
+}
+
+func assertReeBasics[E extensions.RunEndsType](t *testing.T, runEndsType E) {
+	typ := extensions.NewTimestampWithOffsetTypeRunEndEncoded(testTimeUnit, runEndsType)
+
+	assert.Equal(t, "arrow.timestamp_with_offset", typ.ExtensionName())
+	assert.True(t, typ.ExtensionEquals(typ))
+
+	assert.True(t, arrow.TypeEqual(typ, typ))
+	assert.True(t, arrow.TypeEqual(
+		arrow.StructOf(
+			arrow.Field{
+				Name: "timestamp",
+				Type: &arrow.TimestampType{
+					Unit:     testTimeUnit,
+					TimeZone: "UTC",
+				},
+				Nullable: false,
+			},
+			arrow.Field{
+				Name:     "offset_minutes",
+				Type:     ree(arrow.DataType(runEndsType)),
+				Nullable: false,
+			},
+		),
+		typ.StorageType()))
+
+	assert.Equal(t, "extension<arrow.timestamp_with_offset>", typ.String())
+}
+
+func TestTimestampWithOffsetTypeRunEndEncodedBasics(t *testing.T) {
+	assertReeBasics(t, &arrow.Int16Type{})
+	assertReeBasics(t, &arrow.Int32Type{})
+	assertReeBasics(t, &arrow.Int64Type{})
+}
+
+func TestTimestampWithOffsetEquals(t *testing.T) {
+	// Completely different types are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewBool8Type()))
+
+	// Different time units are not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Second)))
+
+	// Different underlying storage type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+
+	// Dict-encoding key type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+
+	// REE index type is not equal
+	assert.False(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+
+	// Equals OK
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Nanosecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Nanosecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetType(arrow.Microsecond).ExtensionEquals(extensions.NewTimestampWithOffsetType(arrow.Microsecond)))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeDictionaryEncoded(arrow.Microsecond, &arrow.Uint16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int16Type{})))
+	assert.True(t, extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{}).ExtensionEquals(extensions.NewTimestampWithOffsetTypeRunEndEncoded(arrow.Microsecond, &arrow.Int32Type{})))
+}
+
+func TestTimestampWithOffsetExtensionBuilder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, _ := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+
+			builder.Append(testDate0)
+			builder.AppendNull()
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			builder.Append(epoch)
+
+			// it should build the array with the correct size
+			arr := builder.NewArray()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+			assert.Equal(t, 5, arr.Data().Len())
+			defer arr.Release()
+
+			// typedArr.Value(i) should return values adjusted for their original timezone
+			assert.Equal(t, testDate0, typedArr.Value(0))
+			assert.Equal(t, testDate1, typedArr.Value(2))
+			assert.Equal(t, testDate2, typedArr.Value(3))
+			assert.Equal(t, epoch, typedArr.Value(4))
+
+			// storage TimeUnit should be the same as we pass in to the builder, and storage timezone should be UTC
+			timestampStructField := typedArr.Storage().(*array.Struct).Field(0)
+			timestampStructDataType := timestampStructField.DataType().(*arrow.TimestampType)
+			assert.Equal(t, timestampStructDataType.Unit, testTimeUnit)
+			assert.Equal(t, timestampStructDataType.TimeZone, "UTC")
+
+			// stored values should be equivalent to the raw values in UTC
+			timestampsArr := timestampStructField.(*array.Timestamp)
+			assert.Equal(t, testDate0.In(time.UTC), timestampsArr.Value(0).ToTime(testTimeUnit))
+			assert.Equal(t, testDate1.In(time.UTC), timestampsArr.Value(2).ToTime(testTimeUnit))
+			assert.Equal(t, testDate2.In(time.UTC), timestampsArr.Value(3).ToTime(testTimeUnit))
+			assert.Equal(t, epoch.In(time.UTC), timestampsArr.Value(4).ToTime(testTimeUnit))
+
+			// the array should encode itself as JSON and string
+			arrStr := arr.String()
+			assert.Equal(t, fmt.Sprintf(`["%[1]s" (null) "%[2]s" "%[3]s" "%[4]s"]`, testDate0, testDate1, testDate2, epoch), arrStr)
+			jsonStr, err := json.Marshal(arr)
+			assert.NoError(t, err)
+
+			// roundtripping from JSON with array.FromJSON should work
+			expectedDataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			roundtripped, _, err := array.FromJSON(mem, expectedDataType, bytes.NewReader(jsonStr))
+			defer roundtripped.Release()
+			assert.NoError(t, err)
+			assert.Truef(t, array.Equal(arr, roundtripped), "expected %s\n\ngot %s", arr, roundtripped)
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesLeadingNull(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			// A leading null as the very first operation on a fresh builder must not
+			// corrupt the offsets. Reading every value below would panic on a
+			// run-end encoded array whose run-ends and values children fell out of
+			// sync (which happened when a run was continued before one existed).
+			builder.AppendValues([]time.Time{epoch, testDate1, testDate2}, []bool{false, true, true})
+
+			arr := builder.NewArray()
+			defer arr.Release()
+			typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+			require.Equal(t, 3, arr.Data().Len())
+			assert.True(t, typedArr.IsNull(0))
+			assert.Equal(t, testDate1, typedArr.Value(1))
+			assert.Equal(t, testDate2, typedArr.Value(2))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderRunEndEncodedNullContinuesRun(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// The same offset surrounds a null. The null must continue the current run
+	// rather than reset the tracked offset, otherwise one logical run is split
+	// into two adjacent runs that share the same value.
+	builder.AppendValues([]time.Time{testDate1, epoch, testDate1}, []bool{true, false, true})
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	offsets := typedArr.Storage().(*array.Struct).Field(1).(*array.RunEndEncoded)
+	assert.Equal(t, 1, offsets.Values().Len())
+
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesNilValids(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// nil and empty validity slices must both be treated as all-valid so the
+	// parent struct and its child builders stay the same length.
+	validsVariants := map[string][]bool{"nil": nil, "empty": {}}
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		for validsName, valids := range validsVariants {
+			t.Run(name+"/"+validsName, func(t *testing.T) {
+				builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+				require.NoError(t, err)
+
+				builder.AppendValues([]time.Time{testDate0, testDate1, testDate2}, valids)
+
+				arr := builder.NewArray()
+				defer arr.Release()
+				typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+				require.Equal(t, 3, arr.Data().Len())
+				assert.Equal(t, testDate0, typedArr.Value(0))
+				assert.Equal(t, testDate1, typedArr.Value(1))
+				assert.Equal(t, testDate2, typedArr.Value(2))
+			})
+		}
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendValuesMismatchedValidsPanics(t *testing.T) {
+	builder, err := extensions.NewTimestampWithOffsetBuilder(memory.DefaultAllocator, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	defer builder.Release()
+
+	assert.Panics(t, func() {
+		builder.AppendValues([]time.Time{testDate0, testDate1}, []bool{true})
+	})
+}
+
+func TestTimestampWithOffsetBuilderReuseAfterNewArray(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, offsetType)
+			require.NoError(t, err)
+
+			builder.Append(testDate1)
+			arr1 := builder.NewArray()
+			defer arr1.Release()
+
+			// Reusing the builder must start a fresh run even when the first value
+			// repeats the previous array's offset; otherwise a run-end-encoded
+			// offset is continued on a freshly reset (empty) builder.
+			builder.Append(testDate1)
+			builder.Append(testDate2)
+			arr2 := builder.NewArray()
+			defer arr2.Release()
+
+			typedArr := arr2.(*extensions.TimestampWithOffsetArray)
+			require.Equal(t, 2, arr2.Data().Len())
+			assert.Equal(t, testDate1, typedArr.Value(0))
+			assert.Equal(t, testDate2, typedArr.Value(1))
+		})
+	}
+}
+
+func TestTimestampWithOffsetBuilderAppendNullBetweenEqualOffsets(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(arrow.PrimitiveTypes.Int16))
+	require.NoError(t, err)
+
+	// AppendNull appends a null offset run; the following value repeats the
+	// pre-null offset and must start its own run instead of being folded into
+	// the null run (which would decode the trailing value's offset as null).
+	builder.Append(testDate1)
+	builder.AppendNull()
+	builder.Append(testDate1)
+
+	arr := builder.NewArray()
+	defer arr.Release()
+	typedArr := arr.(*extensions.TimestampWithOffsetArray)
+
+	require.Equal(t, 3, arr.Data().Len())
+	assert.Equal(t, testDate1, typedArr.Value(0))
+	assert.True(t, typedArr.IsNull(1))
+	assert.Equal(t, testDate1, typedArr.Value(2))
+}
+
+func TestTimestampWithOffsetRunEndEncodedSliced(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	for _, runEnds := range []arrow.DataType{
+		arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64,
+	} {
+		t.Run(runEnds.String(), func(t *testing.T) {
+			builder, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, ree(runEnds))
+			require.NoError(t, err)
+
+			// Offsets [0,0,-510,-510,-510,660,0] give run-ends [2,5,6,7], so a
+			// slice starting at index 3 begins inside a later run: the case where
+			// an offset-unaware physical index decodes the wrong run.
+			values := []time.Time{testDate0, testDate0, testDate1, testDate1, testDate1, testDate2, epoch}
+			builder.AppendValues(values, nil)
+			arr := builder.NewArray().(*extensions.TimestampWithOffsetArray)
+			defer arr.Release()
+
+			want := arr.Values()
+
+			for start := 0; start < arr.Len(); start++ {
+				for end := start; end <= arr.Len(); end++ {
+					sliced := array.NewSlice(arr, int64(start), int64(end)).(*extensions.TimestampWithOffsetArray)
+					got := sliced.Values()
+					require.Equalf(t, end-start, len(got), "slice [%d:%d]", start, end)
+					for k := 0; k < end-start; k++ {
+						require.Truef(t, want[start+k].Equal(got[k]), "slice [%d:%d] row %d instant: want %s got %s", start, end, k, want[start+k], got[k])
+						_, wantOff := want[start+k].Zone()
+						_, gotOff := got[k].Zone()
+						require.Equalf(t, wantOff, gotOff, "slice [%d:%d] row %d offset", start, end, k)
+					}
+					sliced.Release()
+				}
+			}
+		})
+	}
+}
+
+func TestTimestampWithOffsetSubHourNegativeZone(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	// A negative offset smaller than one hour must keep its sign in the zone
+	// name (regression: -30 minutes formatted as "UTC+00:30" instead of "UTC-00:30").
+	b, err := extensions.NewTimestampWithOffsetBuilder(mem, testTimeUnit, arrow.PrimitiveTypes.Int16)
+	require.NoError(t, err)
+	b.Append(testDate0.In(time.FixedZone("UTC-00:30", -30*60)))
+	arr := b.NewArray().(*extensions.TimestampWithOffsetArray)
+	defer arr.Release()
+
+	name, off := arr.Value(0).Zone()
+	require.Equal(t, -30*60, off)
+	require.Equal(t, "UTC-00:30", name)
+}
+
+func TestTimestampWithOffsetExtensionRecordBuilder(t *testing.T) {
+	for name, offsetType := range allAllowedOffsetTypes {
+		t.Run(name, func(t *testing.T) {
+			dataType, _ := extensions.NewTimestampWithOffsetTypeCustomOffset(testTimeUnit, offsetType)
+			schema := arrow.NewSchema([]arrow.Field{
+				{
+					Name:     "timestamp_with_offset",
+					Nullable: true,
+					Type:     dataType,
+				},
+			}, nil)
+			builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
+			defer builder.Release()
+
+			fieldBuilder := builder.Field(0).(*extensions.TimestampWithOffsetBuilder)
+
+			// append a simple time.Time
+			fieldBuilder.Append(testDate0)
+
+			// append the epoch
+			fieldBuilder.Append(epoch)
+
+			// append a null and 2 time.Time all at once
+			values := []time.Time{
+				time.Unix(0, 0).In(time.UTC),
+				testDate1,
+				testDate2,
+			}
+			valids := []bool{false, true, true}
+			fieldBuilder.AppendValues(values, valids)
+
+			// append a value from RFC3339 string
+			fieldBuilder.AppendValueFromString(testDate0.Format(time.RFC3339))
+
+			// append value formatted in a different string layout
+			fieldBuilder.Layout = time.RFC3339Nano
+			fieldBuilder.AppendValueFromString(testDate1.Format(time.RFC3339Nano))
+
+			record := builder.NewRecordBatch()
+
+			// Record batch should JSON-encode values containing per-row timezone info
+			json, err := record.MarshalJSON()
+			require.NoError(t, err)
+			expect := `[{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"1970-01-01T00:00:00Z"}
+,{"timestamp_with_offset":null}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+,{"timestamp_with_offset":"2025-01-01T11:00:00+11:00"}
+,{"timestamp_with_offset":"2025-01-01T00:00:00Z"}
+,{"timestamp_with_offset":"2024-12-31T15:30:00-08:30"}
+]`
+			require.Equal(t, expect, string(json))
+
+			// Record batch roundtrip to JSON should work
+			roundtripped, _, err := array.RecordFromJSON(memory.DefaultAllocator, schema, bytes.NewReader(json))
+			require.NoError(t, err)
+			defer roundtripped.Release()
+			require.Equal(t, schema, roundtripped.Schema())
+			// Avoid array.RecordEqual here: it compares the inner non-nullable

Review Comment:
   Can we add a FIXME here to revisit? Ideally the records SHOULD equal lol



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org