You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2026/07/27 20:01:12 UTC
(arrow-go) branch main updated: fix(arrow/csv): respect timestamp timezones (#1025)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new a1637095 fix(arrow/csv): respect timestamp timezones (#1025)
a1637095 is described below
commit a163709505e7a430baac1ad5c87b513db65d97f7
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Mon Jul 27 22:01:07 2026 +0200
fix(arrow/csv): respect timestamp timezones (#1025)
### Rationale for this change
Timestamp arrays can declare a timezone, but the CSV writer formats
every value directly in UTC. This produces the wrong local timestamp and
also lets invalid timezone declarations pass silently.
### What changes are included in this PR?
* Use the timestamp type conversion function so values are formatted in
the declared timezone.
* Return invalid timezone errors from Writer.Write.
* Propagate conversion errors from nested list values.
### Are these changes tested?
Yes. The tests verify epoch formatting in America/New_York and rejection
of an invalid timezone. All CSV tests and the assertion build pass with
the required Parquet test data.
---
arrow/csv/transformer.go | 19 ++++++++++++++-----
arrow/csv/writer.go | 5 ++++-
arrow/csv/writer_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 61 insertions(+), 6 deletions(-)
diff --git a/arrow/csv/transformer.go b/arrow/csv/transformer.go
index 807050c8..5bcf26e0 100644
--- a/arrow/csv/transformer.go
+++ b/arrow/csv/transformer.go
@@ -27,11 +27,11 @@ import (
"github.com/apache/arrow-go/v18/arrow/array"
)
-func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, stringsReplacer func(string) string) []string {
+func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, stringsReplacer func(string) string) ([]string, error) {
if w.customTypeConverter != nil {
result, handled := w.customTypeConverter(typ, col)
if handled {
- return result
+ return result, nil
}
}
@@ -185,9 +185,13 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, st
case *arrow.TimestampType:
arr := col.(*array.Timestamp)
t := typ.(*arrow.TimestampType)
+ toTime, err := t.GetToTimeFunc()
+ if err != nil {
+ return nil, fmt.Errorf("arrow/csv: invalid timestamp timezone: %w", err)
+ }
for i := 0; i < arr.Len(); i++ {
if arr.IsValid(i) {
- res[i] = arr.Value(i).ToTime(t.Unit).Format("2006-01-02 15:04:05.999999999")
+ res[i] = toTime(arr.Value(i)).Format("2006-01-02 15:04:05.999999999")
} else {
res[i] = w.nullValue
}
@@ -227,7 +231,12 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, st
var b bytes.Buffer
b.Write([]byte{'{'})
writer := csv.NewWriter(&b)
- writer.Write(w.transformColToStringArr(list.DataType(), list, stringsReplacer))
+ values, err := w.transformColToStringArr(list.DataType(), list, stringsReplacer)
+ if err != nil {
+ list.Release()
+ return nil, err
+ }
+ writer.Write(values)
writer.Flush()
b.Truncate(b.Len() - 1)
b.Write([]byte{'}'})
@@ -277,5 +286,5 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, st
default:
panic(fmt.Errorf("arrow/csv: field has unsupported data type %s", typ.String()))
}
- return res
+ return res, nil
}
diff --git a/arrow/csv/writer.go b/arrow/csv/writer.go
index f4891efd..5165d517 100644
--- a/arrow/csv/writer.go
+++ b/arrow/csv/writer.go
@@ -84,7 +84,10 @@ func (w *Writer) Write(record arrow.RecordBatch) error {
}
for j, col := range record.Columns() {
- rows := w.transformColToStringArr(w.schema.Field(j).Type, col, w.stringReplacer)
+ rows, err := w.transformColToStringArr(w.schema.Field(j).Type, col, w.stringReplacer)
+ if err != nil {
+ return err
+ }
for i, row := range rows {
recs[i][j] = row
}
diff --git a/arrow/csv/writer_test.go b/arrow/csv/writer_test.go
index 1701f999..66c664f8 100644
--- a/arrow/csv/writer_test.go
+++ b/arrow/csv/writer_test.go
@@ -664,3 +664,46 @@ func TestCustomTypeConversion(t *testing.T) {
require.Equal(t, expected, buf.String())
}
+
+func TestCSVWriterUsesTimestampTimezone(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ timestampType := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "America/New_York"}
+ builder := array.NewTimestampBuilder(mem, timestampType)
+ builder.Append(0)
+ values := builder.NewTimestampArray()
+ builder.Release()
+ defer values.Release()
+
+ schema := arrow.NewSchema([]arrow.Field{{Name: "timestamp", Type: timestampType}}, nil)
+ record := array.NewRecordBatch(schema, []arrow.Array{values}, 1)
+ defer record.Release()
+
+ var output bytes.Buffer
+ writer := csv.NewWriter(&output, schema)
+ require.NoError(t, writer.Write(record))
+ require.NoError(t, writer.Flush())
+ assert.Equal(t, "1969-12-31 19:00:00\n", output.String())
+}
+
+func TestCSVWriterRejectsInvalidTimestampTimezone(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ timestampType := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "invalid/timezone"}
+ builder := array.NewTimestampBuilder(mem, timestampType)
+ builder.Append(0)
+ values := builder.NewTimestampArray()
+ builder.Release()
+ defer values.Release()
+
+ schema := arrow.NewSchema([]arrow.Field{{Name: "timestamp", Type: timestampType}}, nil)
+ record := array.NewRecordBatch(schema, []arrow.Array{values}, 1)
+ defer record.Release()
+
+ writer := csv.NewWriter(io.Discard, schema)
+ err := writer.Write(record)
+ require.Error(t, err)
+ assert.ErrorContains(t, err, "invalid timestamp timezone")
+}