You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2025/06/16 04:37:19 UTC
(arrow-adbc) branch main updated: fix(go/adbc/driver/snowflake): Adjust the precision of the Timestamp values in the JSON-only path (#2965)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new c3b915a11 fix(go/adbc/driver/snowflake): Adjust the precision of the Timestamp values in the JSON-only path (#2965)
c3b915a11 is described below
commit c3b915a11bc4a294a557b61bf799b07ea8b3c0c8
Author: davidhcoe <13...@users.noreply.github.com>
AuthorDate: Mon Jun 16 00:37:13 2025 -0400
fix(go/adbc/driver/snowflake): Adjust the precision of the Timestamp values in the JSON-only path (#2965)
After the update to adjust the Timestamp precision, the JSON path (which
is hit when using SHOW commands) was still using nanoseconds even though
the schema was specifying the precision as microseconds, resulting in
downstream C# callers getting the error
```
System.ArgumentOutOfRangeException: Ticks must be between DateTime.MinValue.Ticks and DateTime.MaxValue.Ticks.
```
---------
Co-authored-by: David Coe <>
---
csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs | 4 +
.../ColumnNetTypeArrowTypeValue.cs | 35 ++++++++
csharp/test/Drivers/Interop/Snowflake/CastTests.cs | 23 +++---
.../test/Drivers/Interop/Snowflake/ClientTests.cs | 63 +++++++++++++++
go/adbc/driver/snowflake/driver_test.go | 92 ++++++++++++++++++++++
go/adbc/driver/snowflake/record_reader.go | 12 ++-
6 files changed, 215 insertions(+), 14 deletions(-)
diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
index 0289e4ae2..ac2d4159f 100644
--- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
+++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
@@ -270,6 +270,10 @@ namespace Apache.Arrow.Adbc.Tests
byte[]? expectedBytes = ctv.ExpectedValue as byte[];
Assert.True(expectedBytes != null && actualBytes.SequenceEqual(expectedBytes), Utils.FormatMessage($"byte[] values do not match expected values for {ctv.Name} for query [{query}]", environmentName));
}
+ else if (ctv.IsCalculatedResult)
+ {
+ Assert.True(ctv.IsValid(value), Utils.FormatMessage($"Actual value [{value}] for {ctv.Name} does not return true for the IsValid function for query [{query}]", environmentName));
+ }
else if (ctv.ExpectedValue is null)
{
Assert.True(value is null, Utils.FormatMessage($"Expected value [{ctv.ExpectedValue}] does not match actual value [{value}] for {ctv.Name} for query [{query}]", environmentName));
diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ColumnNetTypeArrowTypeValue.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ColumnNetTypeArrowTypeValue.cs
index 8f9af224b..7ba242d4c 100644
--- a/csharp/test/Apache.Arrow.Adbc.Tests/ColumnNetTypeArrowTypeValue.cs
+++ b/csharp/test/Apache.Arrow.Adbc.Tests/ColumnNetTypeArrowTypeValue.cs
@@ -24,6 +24,15 @@ namespace Apache.Arrow.Adbc.Tests
/// </summary>
public class ColumnNetTypeArrowTypeValue
{
+ private Func<object?, bool>? _isValid;
+
+ /// <summary>
+ /// Instantiates <see cref="ColumnNetTypeArrowTypeValue"/>.
+ /// </summary>
+ /// <param name="name">The column name</param>
+ /// <param name="expectedNetType">The expected .NET type</param>
+ /// <param name="expectedArrowArrayType">The expected Arrow type</param>
+ /// <param name="expectedValue">The expected value</param>
public ColumnNetTypeArrowTypeValue(string name, Type expectedNetType, Type expectedArrowArrayType, object? expectedValue)
{
this.Name = name;
@@ -32,6 +41,25 @@ namespace Apache.Arrow.Adbc.Tests
this.ExpectedValue = expectedValue;
}
+ /// <summary>
+ /// Instantiates <see cref="ColumnNetTypeArrowTypeValue"/>.
+ /// </summary>
+ /// <param name="name">The column name</param>
+ /// <param name="expectedNetType">The expected .NET type</param>
+ /// <param name="expectedArrowArrayType">The expected Arrow type</param>
+ /// <param name="expectedValue">The expected value</param>
+ /// <param name="isValid">A function that can be run to compare values</param>
+ /// <exception cref="ArgumentNullException"></exception>
+ public ColumnNetTypeArrowTypeValue(string name, Type expectedNetType, Type expectedArrowArrayType, bool expectedValue, Func<object?, bool> isValid)
+ {
+ this.Name = name;
+ this.ExpectedNetType = expectedNetType;
+ this.ExpectedArrowArrayType = expectedArrowArrayType;
+ this.ExpectedValue = expectedValue;
+
+ _isValid = isValid ?? throw new ArgumentNullException(nameof(isValid));
+ }
+
public string Name { get; set; }
public Type ExpectedNetType { get; set; }
@@ -39,5 +67,12 @@ namespace Apache.Arrow.Adbc.Tests
public Type ExpectedArrowArrayType { get; set; }
public object? ExpectedValue { get; set; }
+
+ public bool IsCalculatedResult => _isValid != null;
+
+ public bool IsValid(object? value)
+ {
+ return _isValid == null ? false : Convert.ToBoolean(this.ExpectedValue) == _isValid(value);
+ }
}
}
diff --git a/csharp/test/Drivers/Interop/Snowflake/CastTests.cs b/csharp/test/Drivers/Interop/Snowflake/CastTests.cs
index dd5ab2e7d..926eea11a 100644
--- a/csharp/test/Drivers/Interop/Snowflake/CastTests.cs
+++ b/csharp/test/Drivers/Interop/Snowflake/CastTests.cs
@@ -310,42 +310,42 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
for (int i = 0; i < doubleArray.Length; i++)
{
Assert.Equal(Convert.ToDouble(value), doubleArray.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Int32:
var int32Array = (Int32Array)valueArray;
for (int i = 0; i < int32Array.Length; i++)
{
Assert.Equal(value, int32Array.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Int64:
var int64Array = (Int64Array)valueArray;
for (int i = 0; i < int64Array.Length; i++)
{
Assert.Equal(value, int64Array.GetValue(i));
- };
+ }
break;
case ArrowTypeId.String:
var stringArray = (StringArray)valueArray;
for (int i = 0; i < stringArray.Length; i++)
{
Assert.Equal(value, stringArray.GetString(i));
- };
+ }
break;
case ArrowTypeId.Boolean:
var booleanArray = (BooleanArray)valueArray;
for (int i = 0; i < booleanArray.Length; i++)
{
Assert.Equal(value, booleanArray.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Date64:
var date64Array = (Date64Array)valueArray;
for (int i = 0; i < date64Array.Length; i++)
{
Assert.Equal(value, date64Array.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Decimal128:
var decimal128Array = (Decimal128Array)valueArray;
@@ -353,21 +353,21 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
{
Assert.Equal(value == null ? null : Convert.ToDecimal(value), decimal128Array.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Decimal256:
var decimal256Array = (Decimal256Array)valueArray;
for (int i = 0; i < decimal256Array.Length; i++)
{
Assert.Equal(value, decimal256Array.GetValue(i));
- };
+ }
break;
case ArrowTypeId.Timestamp:
var timestampArray = (TimestampArray)valueArray;
for (int i = 0; i < timestampArray.Length; i++)
{
Assert.Equal(value == null ? null : new DateTimeOffset(Convert.ToDateTime(value).ToUniversalTime()), timestampArray.GetTimestamp(i));
- };
+ }
break;
case ArrowTypeId.Time64:
var time64Array = (Time64Array)valueArray;
@@ -378,16 +378,15 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
#else
Assert.Equal(Convert.ToInt64(value), time64Array.GetMicroSeconds(i));
#endif
- };
+ }
break;
case ArrowTypeId.Date32:
var date32Array = (Date32Array)valueArray;
for (int i = 0; i < date32Array.Length; i++)
{
Assert.Equal(Convert.ToDateTime(value), date32Array.GetDateTimeOffset(i));
- };
+ }
break;
-
default:
throw new ArgumentException(string.Format("Unexpected ArrowTypeId: {0}({1})", field.DataType.TypeId.ToString(), field.DataType.TypeId));
diff --git a/csharp/test/Drivers/Interop/Snowflake/ClientTests.cs b/csharp/test/Drivers/Interop/Snowflake/ClientTests.cs
index 93ad04882..128f1d87b 100644
--- a/csharp/test/Drivers/Interop/Snowflake/ClientTests.cs
+++ b/csharp/test/Drivers/Interop/Snowflake/ClientTests.cs
@@ -280,6 +280,69 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
Assert.Throws<Exception>(() => ValidateTimestampPrecision(SnowflakeConstants.OptionValueNanosecondsNoOverflow, query, expectedNanoseconddValues));
}
+ [SkippableFact, Order(6)]
+ public void VerifyTimestampPrecisionJson()
+ {
+ SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
+ testConfiguration.MaxTimestampPrecision = "microseconds";
+
+ string tempTable = "pk_" + Guid.NewGuid().ToString().Replace("-", "");
+
+ using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration))
+ {
+ SampleDataBuilder sampleDataBuilder = new SampleDataBuilder();
+ sampleDataBuilder.Samples.Add(
+ new SampleData()
+ {
+ // create the table
+ PreQueryCommands = new List<string> {
+ @$"CREATE OR REPLACE TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}(
+ id INT PRIMARY KEY,
+ name STRING
+ );"
+ },
+ // run the SHOW PRIMARY KEYS command, which returns data as json
+ Query = $"SHOW PRIMARY KEYS IN TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}",
+ ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
+ {
+ new ColumnNetTypeArrowTypeValue("created_on", typeof(DateTimeOffset), typeof(TimestampType), true,
+ v =>
+ {
+ if (v is DateTimeOffset createdOn)
+ {
+ // the key will have just been created, so just compare that the two times
+ // are only a few minutes apart
+ DateTimeOffset now = DateTimeOffset.UtcNow;
+ TimeSpan difference = now - createdOn;
+ return difference.Duration() < TimeSpan.FromMinutes(2);
+ }
+ else
+ {
+ return false;
+ }
+ }),
+ new ColumnNetTypeArrowTypeValue("database_name", typeof(string), typeof(StringType), testConfiguration.Metadata.Catalog),
+ new ColumnNetTypeArrowTypeValue("schema_name", typeof(string), typeof(StringType), testConfiguration.Metadata.Schema),
+ new ColumnNetTypeArrowTypeValue("table_name", typeof(string), typeof(StringType), tempTable.ToUpper()),
+ new ColumnNetTypeArrowTypeValue("column_name", typeof(string), typeof(StringType), "ID"),
+ new ColumnNetTypeArrowTypeValue("key_sequence", typeof(SqlDecimal), typeof(Decimal128Type), new SqlDecimal(1m)),
+
+ // we dont control these, but also don't care about the value
+ new ColumnNetTypeArrowTypeValue("constraint_name", typeof(string), typeof(StringType), true, v => { return true; }),
+ new ColumnNetTypeArrowTypeValue("rely", typeof(string), typeof(StringType), true, v => { return true; }),
+ new ColumnNetTypeArrowTypeValue("comment", typeof(string), typeof(StringType), null)
+ },
+ // drop the table
+ PostQueryCommands = new List<string>() {
+ $"DROP TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}"
+ }
+ });
+
+ Tests.ClientTests.VerifyTypesAndValues(adbcConnection, sampleDataBuilder);
+ }
+ }
+
+
private void ValidateTimestampPrecision(string precision, string query, List<ColumnNetTypeArrowTypeValue> expectedValues)
{
SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
diff --git a/go/adbc/driver/snowflake/driver_test.go b/go/adbc/driver/snowflake/driver_test.go
index 4a899d082..8b013c1fd 100644
--- a/go/adbc/driver/snowflake/driver_test.go
+++ b/go/adbc/driver/snowflake/driver_test.go
@@ -1724,6 +1724,98 @@ func (suite *SnowflakeTests) TestBooleanType() {
}
}
+func (suite *SnowflakeTests) TestTimestampPrecisionJson() {
+ opts := suite.Quirks.DatabaseOptions()
+ opts[driver.OptionMaxTimestampPrecision] = "microseconds"
+
+ db, err := suite.driver.NewDatabase(opts)
+ suite.NoError(err)
+ defer validation.CheckedClose(suite.T(), db)
+ cnxn, err := db.Open(suite.ctx)
+ suite.NoError(err)
+ defer validation.CheckedClose(suite.T(), cnxn)
+ stmt, _ := cnxn.NewStatement()
+
+ id := uuid.New()
+ tempTable := "pk_" + strings.ReplaceAll(id.String(), "-", "")
+
+ query := fmt.Sprintf(`CREATE OR REPLACE TABLE %s.%s.%s (
+ id INT PRIMARY KEY,
+ name STRING);`, suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)
+
+ suite.Require().NoError(stmt.SetSqlQuery(query))
+ _, err = stmt.ExecuteUpdate(suite.ctx)
+ suite.Require().NoError(err)
+ suite.Require().NoError(stmt.Close())
+
+ query = fmt.Sprintf("SHOW PRIMARY KEYS IN TABLE %s.%s.%s", suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)
+ stmt, _ = cnxn.NewStatement()
+ suite.Require().NoError(stmt.SetSqlQuery(query))
+ rdr, _, err := stmt.ExecuteQuery(suite.ctx)
+ defer rdr.Release()
+ suite.Require().NoError(err)
+
+ suite.True(rdr.Next())
+ rec := rdr.Record()
+
+ suite.Equal(1, int(rec.NumRows()))
+
+ // Get column indexes
+ getColIdx := func(name string) int {
+ for i := 0; i < int(rec.NumCols()); i++ {
+ if rec.ColumnName(i) == name {
+ return i
+ }
+ }
+ panic("Column not found: " + name)
+ }
+
+ // Expected column names
+ dbIdx := getColIdx("database_name")
+ schemaIdx := getColIdx("schema_name")
+ tableIdx := getColIdx("table_name")
+ colIdx := getColIdx("column_name")
+ seqIdx := getColIdx("key_sequence")
+ createdIdx := getColIdx("created_on")
+
+ i := 0
+ dbName := rec.Column(dbIdx).(*array.String).Value(i)
+ schema := rec.Column(schemaIdx).(*array.String).Value(i)
+ tbl := rec.Column(tableIdx).(*array.String).Value(i)
+ column := rec.Column(colIdx).(*array.String).Value(i)
+ keySeq := rec.Column(seqIdx).(*array.Int64).Value(i)
+
+ // Created_on should be a timestamp array
+ createdCol := rec.Column(createdIdx).(*array.Timestamp)
+ created := time.Unix(0, int64(createdCol.Value(i))*int64(time.Microsecond)).UTC()
+
+ // Perform checks
+ if strings.EqualFold(dbName, suite.Quirks.catalogName) &&
+ strings.EqualFold(schema, suite.Quirks.schemaName) &&
+ strings.EqualFold(tbl, tempTable) &&
+ strings.EqualFold(column, "id") &&
+ keySeq == 1 {
+
+ now := time.Now().UTC()
+ diff := now.Sub(created)
+ if diff < 0 {
+ diff = -diff
+ }
+ // since this was just created, make sure the times are within a short difference
+ suite.Assert().True(diff <= 2*time.Minute)
+ } else {
+ panic("Invalid values")
+ }
+ suite.Require().NoError(stmt.Close())
+
+ query = fmt.Sprintf("DROP TABLE %s.%s.%s", suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)
+ stmt, _ = cnxn.NewStatement()
+ suite.Require().NoError(stmt.SetSqlQuery(query))
+ _, err = stmt.ExecuteUpdate(suite.ctx)
+ suite.Require().NoError(err)
+ suite.Require().NoError(stmt.Close())
+}
+
func (suite *SnowflakeTests) TestTimestampPrecision() {
query_ntz := "select TO_TIMESTAMP_NTZ('0001-01-01 00:00:00.000000000') as Jan01_0001, TO_TIMESTAMP_NTZ('2025-06-02 10:37:56.123456789') as June02_2025, TO_TIMESTAMP_NTZ('9999-12-31 23:59:59.999999999') As December31_9999"
diff --git a/go/adbc/driver/snowflake/record_reader.go b/go/adbc/driver/snowflake/record_reader.go
index 7c347b922..140b1050a 100644
--- a/go/adbc/driver/snowflake/record_reader.go
+++ b/go/adbc/driver/snowflake/record_reader.go
@@ -479,8 +479,16 @@ func jsonDataToArrow(_ context.Context, bldr *array.RecordBuilder, rawData [][]*
return nil, err
}
- fb.Append(arrow.Timestamp(sec*1e9 + nsec))
-
+ if maxTimestampPrecision == Microseconds {
+ tt := time.Unix(sec, nsec)
+ ts, err := getArrowTimestampFromTime(tt, arrow.Microsecond, arrow.Nanosecond, maxTimestampPrecision)
+ if err != nil {
+ return nil, err
+ }
+ fb.Append(ts)
+ } else {
+ fb.Append(arrow.Timestamp(sec*1e9 + nsec))
+ }
case *array.BinaryBuilder:
b, err := hex.DecodeString(*col)
if err != nil {