(arrow-go) branch main updated: fix(parquet/pqarrow): preserve INT96 timestamps across units (#1006)

[ze...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 8c72476d fix(parquet/pqarrow): preserve INT96 timestamps across units (#1006)
8c72476d is described below

commit 8c72476df63df6d3d29b938ca528e1526985370b
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Mon Jul 27 20:57:18 2026 +0200

    fix(parquet/pqarrow): preserve INT96 timestamps across units (#1006)
    
    ### Rationale for this change
    
    INT96 timestamp conversion currently routes values through
    `time.Duration` and applies a nanoseconds-per-day modulus directly to
    the source value. This produces incorrect results for second,
    millisecond, and microsecond timestamps, overflows for timestamps
    outside the duration range, and leaves negative timestamps with a
    negative time-of-day remainder.
    
    ### What changes are included in this PR?
    
    - Convert each Arrow timestamp unit using integer units-per-day and
    nanoseconds-per-unit constants.
    - Normalize negative remainders onto the preceding Julian day.
    - Add focused coverage for every timestamp unit on both sides of the
    Unix epoch.
    
    ### Are these changes tested?
    
    Yes. The focused pqarrow tests pass.
---
 parquet/pqarrow/encode_arrow.go            | 24 ++++++++------
 parquet/pqarrow/encode_arrow_int96_test.go | 51 ++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 9 deletions(-)

diff --git a/parquet/pqarrow/encode_arrow.go b/parquet/pqarrow/encode_arrow.go
index 9ec11ff8..c7f089c2 100644
--- a/parquet/pqarrow/encode_arrow.go
+++ b/parquet/pqarrow/encode_arrow.go
@@ -24,7 +24,6 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"time"
 	"unsafe"
 
 	"github.com/apache/arrow-go/v18/arrow"
@@ -712,20 +711,27 @@ const (
 )
 
 func arrowTimestampToImpalaTimestamp(unit arrow.TimeUnit, t int64, out *parquet.Int96) {
-	var d time.Duration
+	var unitsPerDay, nanosPerUnit int64
 	switch unit {
 	case arrow.Second:
-		d = time.Duration(t) * time.Second
-	case arrow.Microsecond:
-		d = time.Duration(t) * time.Microsecond
+		unitsPerDay, nanosPerUnit = 24*60*60, 1000*1000*1000
 	case arrow.Millisecond:
-		d = time.Duration(t) * time.Millisecond
+		unitsPerDay, nanosPerUnit = 24*60*60*1000, 1000*1000
+	case arrow.Microsecond:
+		unitsPerDay, nanosPerUnit = 24*60*60*1000*1000, 1000
 	case arrow.Nanosecond:
-		d = time.Duration(t) * time.Nanosecond
+		unitsPerDay, nanosPerUnit = nanoSecondsPerDay, 1
+	}
+
+	days := t / unitsPerDay
+	remainder := t % unitsPerDay
+	if remainder < 0 {
+		days--
+		remainder += unitsPerDay
 	}
 
-	julianDays := (int64(d.Hours()) / 24) + julianEpochOffsetDays
-	lastDayNanos := t % (nanoSecondsPerDay)
+	julianDays := days + julianEpochOffsetDays
+	lastDayNanos := remainder * nanosPerUnit
 	binary.LittleEndian.PutUint64((*out)[:8], uint64(lastDayNanos))
 	binary.LittleEndian.PutUint32((*out)[8:], uint32(julianDays))
 }
diff --git a/parquet/pqarrow/encode_arrow_int96_test.go b/parquet/pqarrow/encode_arrow_int96_test.go
new file mode 100644
index 00000000..4e5cbf2d
--- /dev/null
+++ b/parquet/pqarrow/encode_arrow_int96_test.go
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pqarrow
+
+import (
+	"testing"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/parquet"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestArrowTimestampToImpalaTimestamp(t *testing.T) {
+	tests := []struct {
+		name  string
+		unit  arrow.TimeUnit
+		value arrow.Timestamp
+	}{
+		{"seconds after epoch", arrow.Second, 946_684_801},
+		{"seconds before epoch", arrow.Second, -1},
+		{"milliseconds after epoch", arrow.Millisecond, 946_684_800_001},
+		{"milliseconds before epoch", arrow.Millisecond, -1},
+		{"microseconds after epoch", arrow.Microsecond, 946_684_800_000_001},
+		{"microseconds before epoch", arrow.Microsecond, -1},
+		{"nanoseconds after epoch", arrow.Nanosecond, 946_684_800_000_000_001},
+		{"nanoseconds before epoch", arrow.Nanosecond, -1},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var got parquet.Int96
+			arrowTimestampToImpalaTimestamp(tt.unit, int64(tt.value), &got)
+
+			assert.Equal(t, tt.value.ToTime(tt.unit), got.ToTime())
+		})
+	}
+}