You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2025/09/03 23:59:53 UTC
(arrow-adbc) branch main updated: fix(csharp/src/Drivers/Apache/Spark): fix column metadata index offset for Spark standard (#3392)
This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 5238995e7 fix(csharp/src/Drivers/Apache/Spark): fix column metadata index offset for Spark standard (#3392)
5238995e7 is described below
commit 5238995e7521106ee0a7d41333ef07d3f9b7b7a6
Author: Bruce Irschick <br...@improving.com>
AuthorDate: Wed Sep 3 16:59:47 2025 -0700
fix(csharp/src/Drivers/Apache/Spark): fix column metadata index offset for Spark standard (#3392)
fix column metadata index offset for Spark standard.
Previous calls to GetObjects would throw an "index out of range"
exception.
---
csharp/src/Drivers/Apache/Spark/SparkConnection.cs | 2 +
.../Drivers/Apache/Spark/SparkHttpConnection.cs | 2 -
.../Apache/Spark/SparkStandardConnection.cs | 2 -
.../Drivers/Apache/Spark/SparkConnectionTest.cs | 51 +++++++++++++++-------
csharp/test/Drivers/Apache/Spark/StatementTests.cs | 1 +
5 files changed, 39 insertions(+), 19 deletions(-)
diff --git a/csharp/src/Drivers/Apache/Spark/SparkConnection.cs b/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
index bf6269c3a..3b0fb634b 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkConnection.cs
@@ -65,6 +65,8 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
protected internal override int PositionRequiredOffset => 1;
+ protected override int ColumnMapIndexOffset => 1;
+
internal override void SetPrecisionScaleAndTypeName(
short colType,
string typeName,
diff --git a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
index b3710314d..de5492175 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
@@ -245,8 +245,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
internal override SparkServerType ServerType => SparkServerType.Http;
- protected override int ColumnMapIndexOffset => 1;
-
public override string AssemblyVersion => s_assemblyVersion;
public override string AssemblyName => s_assemblyName;
diff --git a/csharp/src/Drivers/Apache/Spark/SparkStandardConnection.cs b/csharp/src/Drivers/Apache/Spark/SparkStandardConnection.cs
index f7438fe47..46faab96e 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkStandardConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkStandardConnection.cs
@@ -218,7 +218,5 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
public override string AssemblyName => s_assemblyName;
public override string AssemblyVersion => s_assemblyVersion;
-
- protected override int ColumnMapIndexOffset => 0;
}
}
diff --git a/csharp/test/Drivers/Apache/Spark/SparkConnectionTest.cs b/csharp/test/Drivers/Apache/Spark/SparkConnectionTest.cs
index 2ca2c1ab6..d1dfa123e 100644
--- a/csharp/test/Drivers/Apache/Spark/SparkConnectionTest.cs
+++ b/csharp/test/Drivers/Apache/Spark/SparkConnectionTest.cs
@@ -290,24 +290,45 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark
{
public InvalidConnectionParametersTestData()
{
+ string serverType = SparkServerTypeConstants.Standard;
Add(new([], typeof(ArgumentException)));
Add(new(new() { [SparkParameters.Type] = " " }, typeof(ArgumentOutOfRangeException)));
Add(new(new() { [SparkParameters.Type] = "xxx" }, typeof(ArgumentOutOfRangeException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Standard }, typeof(ArgumentOutOfRangeException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = " " }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "invalid!server.com" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "http://valid.server.com" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"unknown_auth_type" }, typeof(ArgumentOutOfRangeException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}", [SparkParameters.Token] = "abcdef" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = ((long)int.MaxValue + 1).ToString() }, typeof(ArgumentOutOfRangeException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "non-numeric" }, typeof(ArgumentOutOfRangeException)));
- Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Http, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Standard }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = " " }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "invalid!server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "http://valid.server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"unknown_auth_type" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}", [SparkParameters.Token] = "abcdef" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = ((long)int.MaxValue + 1).ToString() }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "non-numeric" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "" }, typeof(ArgumentException)));
+
+ serverType = SparkServerTypeConstants.Http;
+ Add(new([], typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = " " }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = "xxx" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = SparkServerTypeConstants.Standard }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = " " }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "invalid!server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "http://valid.server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"unknown_auth_type" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Basic}", [SparkParameters.Token] = "abcdef" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [SparkParameters.AuthType] = $"{SparkAuthTypeConstants.Token}", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Password] = "myPassword" }, typeof(ArgumentException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = ((long)int.MaxValue + 1).ToString() }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "non-numeric" }, typeof(ArgumentOutOfRangeException)));
+ Add(new(new() { [SparkParameters.Type] = serverType, [SparkParameters.HostName] = "valid.server.com", [AdbcOptions.Username] = "user", [AdbcOptions.Password] = "myPassword", [SparkParameters.ConnectTimeoutMilliseconds] = "" }, typeof(ArgumentOutOfRangeException)));
}
}
}
diff --git a/csharp/test/Drivers/Apache/Spark/StatementTests.cs b/csharp/test/Drivers/Apache/Spark/StatementTests.cs
index 05613933e..d8eb7d478 100644
--- a/csharp/test/Drivers/Apache/Spark/StatementTests.cs
+++ b/csharp/test/Drivers/Apache/Spark/StatementTests.cs
@@ -42,6 +42,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark
[InlineData(LongRunningStatementTimeoutTestData.LongRunningQuery)]
internal override async Task CanCancelStatementTest(string query)
{
+ Skip.If(TestConfiguration.Type == "standard", "Spark 'standard' transport does not support cancellation test.");
await base.CanCancelStatementTest(query);
}