You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2025/03/25 21:48:57 UTC
(arrow-adbc) branch main updated: feat(csharp/src/Drivers/Apache): Custom ssl server certificate validation for Spark, Impala & Hive (#2610)
This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 152d25cbd feat(csharp/src/Drivers/Apache): Custom ssl server certificate validation for Spark, Impala & Hive (#2610)
152d25cbd is described below
commit 152d25cbdc67730998534284c0b111b988a0e0e7
Author: Sudhir Reddy Emmadi <re...@gmail.com>
AuthorDate: Wed Mar 26 03:18:52 2025 +0530
feat(csharp/src/Drivers/Apache): Custom ssl server certificate validation for Spark, Impala & Hive (#2610)
Co-authored-by: Sudhir Emmadi <em...@microsoft.com>
---
.../Drivers/Apache/Hive2/HiveServer2Connection.cs | 7 +-
.../Apache/Hive2/HiveServer2HttpConnection.cs | 26 +-----
.../Drivers/Apache/Hive2/HiveServer2Parameters.cs | 10 +-
.../src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs | 101 +++++++++++++++++++++
.../Drivers/Apache/Hive2/HiveServer2TlsOption.cs | 53 -----------
csharp/src/Drivers/Apache/Hive2/README.md | 6 +-
.../Drivers/Apache/Impala/ImpalaHttpConnection.cs | 27 +-----
.../Apache/Impala/ImpalaStandardConnection.cs | 1 -
csharp/src/Drivers/Apache/Spark/README.md | 6 +-
.../Drivers/Apache/Spark/SparkHttpConnection.cs | 27 +-----
csharp/src/Drivers/Apache/Spark/SparkParameters.cs | 1 -
.../Apache/Hive2/HiveServer2ParametersTest.cs | 31 -------
.../Apache/Hive2/HiveServer2TestEnvironment.cs | 4 -
.../Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs | 59 ++++++++++++
.../Drivers/Apache/Spark/SparkTestEnvironment.cs | 4 -
15 files changed, 191 insertions(+), 172 deletions(-)
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs b/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
index e182adb4f..ab3efea31 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
@@ -342,7 +342,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
protected internal DataTypeConversion DataTypeConversion { get; set; } = DataTypeConversion.None;
- protected internal HiveServer2TlsOption TlsOptions { get; set; } = HiveServer2TlsOption.Empty;
+ protected internal TlsProperties TlsOptions { get; set; } = new TlsProperties();
protected internal int ConnectTimeoutMilliseconds { get; set; } = ConnectTimeoutMillisecondsDefault;
@@ -731,7 +731,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
return fileVersionInfo.ProductVersion ?? GetProductVersionDefault();
}
- protected static Uri GetBaseAddress(string? uri, string? hostName, string? path, string? port, string hostOptionName)
+ protected static Uri GetBaseAddress(string? uri, string? hostName, string? path, string? port, string hostOptionName, bool isTlsEnabled)
{
// Uri property takes precedent.
if (!string.IsNullOrWhiteSpace(uri))
@@ -755,8 +755,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
bool isPortSet = !string.IsNullOrEmpty(port);
bool isValidPortNumber = int.TryParse(port, out int portNumber) && portNumber > 0;
- bool isDefaultHttpsPort = !isPortSet || (isValidPortNumber && portNumber == 443);
- string uriScheme = isDefaultHttpsPort ? Uri.UriSchemeHttps : Uri.UriSchemeHttp;
+ string uriScheme = isTlsEnabled ? Uri.UriSchemeHttps : Uri.UriSchemeHttp;
int uriPort;
if (!isPortSet)
uriPort = -1;
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs b/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
index 19f799322..187e5712c 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
@@ -121,7 +121,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
Properties.TryGetValue(HiveServer2Parameters.Path, out string? path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port, HiveServer2Parameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port, HiveServer2Parameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -129,8 +129,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
{
Properties.TryGetValue(HiveServer2Parameters.DataTypeConv, out string? dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(HiveServer2Parameters.TLSOptions, out string? tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(HiveServer2Parameters.ConnectTimeoutMilliseconds, out string? connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -138,6 +136,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
? connectTimeoutMsValue
: throw new ArgumentOutOfRangeException(HiveServer2Parameters.ConnectTimeoutMilliseconds, connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 .. {int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
public override AdbcStatement CreateStatement()
@@ -166,10 +165,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port, HiveServer2Parameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port, HiveServer2Parameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue = GetAuthenticationHeaderValue(authTypeValue, username, password);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler = HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
httpClientHandler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
@@ -190,23 +189,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback = (request, certificate, chain, policyErrors) =>
- {
- if (policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) && !TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned)) return false;
- if (policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) && !TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch)) return false;
-
- return true;
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue? GetAuthenticationHeaderValue(HiveServer2AuthType authType, string? username, string? password)
{
if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) && (authType == HiveServer2AuthType.Empty || authType == HiveServer2AuthType.Basic))
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs b/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
index b9d7f2c2a..f1bb90f11 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
@@ -25,7 +25,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
public const string AuthType = "adbc.hive.auth_type";
public const string TransportType = "adbc.hive.transport_type";
public const string DataTypeConv = "adbc.hive.data_type_conv";
- public const string TLSOptions = "adbc.hive.tls_options";
public const string ConnectTimeoutMilliseconds = "adbc.hive.connect_timeout_ms";
}
@@ -47,9 +46,12 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
public const string Scalar = "scalar";
}
- public static class TlsOptions
+ public static class HttpTlsOptions
{
- public const string AllowSelfSigned = "allow_self_signed";
- public const string AllowHostnameMismatch = "allow_hostname_mismatch";
+ public const string IsTlsEnabled = "adbc.http_options.tls.enabled";
+ public const string AllowSelfSigned = "adbc.http_options.tls.allow_self_signed";
+ public const string AllowHostnameMismatch = "adbc.http_options.tls.allow_hostname_mismatch";
+ public const string TrustedCertificatePath = "adbc.http_options.tls.trusted_certificate_path";
+ public const string DisableServerCertificateValidation = "adbc.http_options.tls.disable_server_certificate_validation";
}
}
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs
new file mode 100644
index 000000000..b7e7c8b38
--- /dev/null
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs
@@ -0,0 +1,101 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
+using System.Net.Http;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
+{
+ class TlsProperties
+ {
+ public bool IsTlsEnabled { get; set; }
+ public bool DisableServerCertificateValidation { get; set; }
+ public bool AllowHostnameMismatch { get; set; }
+ public bool AllowSelfSigned { get; set; }
+ public string? TrustedCertificatePath { get; set; }
+ }
+
+ static class HiveServer2TlsImpl
+ {
+ static internal TlsProperties GetHttpTlsOptions(IReadOnlyDictionary<string, string> properties)
+ {
+ TlsProperties tlsProperties = new TlsProperties();
+ if (properties.TryGetValue(AdbcOptions.Uri, out string? uri) && !string.IsNullOrWhiteSpace(uri))
+ {
+ var uriValue = new Uri(uri);
+ tlsProperties.IsTlsEnabled = uriValue.Scheme == Uri.UriSchemeHttps || (properties.TryGetValue(HttpTlsOptions.IsTlsEnabled, out string? isSslEnabled) && bool.TryParse(isSslEnabled, out bool isSslEnabledBool) && isSslEnabledBool);
+ }
+ else
+ {
+ tlsProperties.IsTlsEnabled = properties.TryGetValue(HttpTlsOptions.IsTlsEnabled, out string? isSslEnabled) && bool.TryParse(isSslEnabled, out bool isSslEnabledBool) && isSslEnabledBool;
+ }
+ if (!tlsProperties.IsTlsEnabled)
+ {
+ return tlsProperties;
+ }
+ tlsProperties.IsTlsEnabled = true;
+ if (properties.TryGetValue(HttpTlsOptions.DisableServerCertificateValidation, out string? disableServerCertificateValidation) && bool.TryParse(disableServerCertificateValidation, out bool disableServerCertificateValidationBool) && disableServerCertificateValidationBool)
+ {
+ tlsProperties.DisableServerCertificateValidation = true;
+ return tlsProperties;
+ }
+ tlsProperties.DisableServerCertificateValidation = false;
+ tlsProperties.AllowHostnameMismatch = properties.TryGetValue(HttpTlsOptions.AllowHostnameMismatch, out string? allowHostnameMismatch) && bool.TryParse(allowHostnameMismatch, out bool allowHostnameMismatchBool) && allowHostnameMismatchBool;
+ tlsProperties.AllowSelfSigned = properties.TryGetValue(HttpTlsOptions.AllowSelfSigned, out string? allowSelfSigned) && bool.TryParse(allowSelfSigned, out bool allowSelfSignedBool) && allowSelfSignedBool;
+ if (tlsProperties.AllowSelfSigned)
+ {
+ if (!properties.TryGetValue(HttpTlsOptions.TrustedCertificatePath, out string? trustedCertificatePath)) return tlsProperties;
+ tlsProperties.TrustedCertificatePath = trustedCertificatePath != "" && File.Exists(trustedCertificatePath) ? trustedCertificatePath : throw new FileNotFoundException("Trusted certificate path is invalid or file does not exist.");
+ }
+ return tlsProperties;
+ }
+
+ static internal HttpClientHandler NewHttpClientHandler(TlsProperties tlsProperties)
+ {
+ HttpClientHandler httpClientHandler = new();
+ if (tlsProperties.IsTlsEnabled)
+ {
+ httpClientHandler.ServerCertificateCustomValidationCallback = (request, certificate, chain, policyErrors) =>
+ {
+ if (policyErrors == SslPolicyErrors.None || tlsProperties.DisableServerCertificateValidation) return true;
+ if (string.IsNullOrEmpty(tlsProperties.TrustedCertificatePath))
+ {
+ return
+ (!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) || tlsProperties.AllowSelfSigned)
+ && (!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) || tlsProperties.AllowHostnameMismatch);
+ }
+ if (certificate == null)
+ return false;
+ X509Certificate2 customCertificate = new X509Certificate2(tlsProperties.TrustedCertificatePath);
+ X509Chain chain2 = new X509Chain();
+ chain2.ChainPolicy.ExtraStore.Add(customCertificate);
+
+ // "tell the X509Chain class that I do trust this root certs and it should check just the certs in the chain and nothing else"
+ chain2.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority;
+
+ // Build the chain and verify
+ return chain2.Build(certificate);
+ };
+ }
+ return httpClientHandler;
+ }
+ }
+}
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs
deleted file mode 100644
index 84f56a485..000000000
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-
-namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
-{
- [Flags]
- internal enum HiveServer2TlsOption
- {
- Empty = 0,
- AllowSelfSigned = 1,
- AllowHostnameMismatch = 2,
- }
-
- internal static class TlsOptionsParser
- {
- internal const string SupportedList = TlsOptions.AllowSelfSigned + "," + TlsOptions.AllowHostnameMismatch;
-
- internal static HiveServer2TlsOption Parse(string? tlsOptions)
- {
- HiveServer2TlsOption options = HiveServer2TlsOption.Empty;
- if (tlsOptions == null) return options;
-
- string[] valueList = tlsOptions.Split(',');
- foreach (string tlsOption in valueList)
- {
- options |= (tlsOption?.Trim().ToLowerInvariant()) switch
- {
- null or "" => HiveServer2TlsOption.Empty,
- TlsOptions.AllowSelfSigned => HiveServer2TlsOption.AllowSelfSigned,
- TlsOptions.AllowHostnameMismatch => HiveServer2TlsOption.AllowHostnameMismatch,
- _ => throw new ArgumentOutOfRangeException(nameof(tlsOptions), tlsOption, "Invalid or unsupported TLS option"),
- };
- }
- return options;
- }
- }
-}
diff --git a/csharp/src/Drivers/Apache/Hive2/README.md b/csharp/src/Drivers/Apache/Hive2/README.md
index e9c863190..5e665d3f8 100644
--- a/csharp/src/Drivers/Apache/Hive2/README.md
+++ b/csharp/src/Drivers/Apache/Hive2/README.md
@@ -35,11 +35,15 @@ but can also be passed in the call to `AdbcDatabase.Connect`.
| `username` | The user name used for basic authentication | |
| `password` | The password for the user name used for basic authentication. | |
| `adbc.hive.data_type_conv` | Comma-separated list of data conversion options. Each option indicates the type of conversion to perform on data returned from the Hive server. <br><br>Allowed values: `none`, `scalar`. <br><br>Option `none` indicates there is no conversion from Hive type to native type (i.e., no conversion from String to Timestamp for Apache Hive over HTTP). Example `adbc.hive.conv_data_type=none`. <br><br>Option `scalar` will perform conversion (if necessary) from the Hiv [...]
-| `adbc.hive.tls_options` | Comma-separated list of TLS/SSL options. Each option indicates the TLS/SSL option when connecting to a Hive server. <br><br>Allowed values: `allow_self_signed`, `allow_hostname_mismatch`. <br><br>Option `allow_self_signed` allows certificate errors due to an unknown certificate authority, typically when using a self-signed certificate. Option `allow_hostname_mismatch` allow certificate errors due to a mismatch of the hostname. (e.g., when connecting through an [...]
| `adbc.hive.connect_timeout_ms` | Sets the timeout (in milliseconds) to open a new session. Values can be 0 (infinite) or greater than zero. | `30000` |
| `adbc.apache.statement.batch_size` | Sets the maximum number of rows to retrieve in a single batch request. | `50000` |
| `adbc.apache.statement.polltime_ms` | If polling is necessary to get a result, this option sets the length of time (in milliseconds) to wait between polls. | `500` |
| `adbc.apache.statement.query_timeout_s` | Sets the maximum time (in seconds) for a query to complete. Values can be 0 (infinite) or greater than zero. | `60` |
+| `adbc.http_options.tls.enabled` | If tls needs to enabled or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.disable_server_certificate_validation` | If tls/ssl server certificate validation needs to enabled or not. One of `True`, `False`. If set to True, all certificate validation errors are ignored | `False` |
+| `adbc.http_options.tls.allow_self_signed` | If self signed tls/ssl certificate needs to be allowed or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.allow_hostname_mismatch` | If hostname mismatch is allowed for ssl. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.trusted_certificate_path` | The full path of the tls/ssl certificate .pem file containing custom CA certificates for verifying the server when connecting over TLS | `` |
## Timeout Configuration
diff --git a/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs b/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
index 7de129df7..914ba9269 100644
--- a/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
@@ -105,7 +105,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(ImpalaParameters.Path, out string? path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port, ImpalaParameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port, ImpalaParameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -113,8 +113,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
{
Properties.TryGetValue(ImpalaParameters.DataTypeConv, out string? dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(ImpalaParameters.TLSOptions, out string? tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(ImpalaParameters.ConnectTimeoutMilliseconds, out string? connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -122,6 +120,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
? connectTimeoutMsValue
: throw new ArgumentOutOfRangeException(ImpalaParameters.ConnectTimeoutMilliseconds, connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 .. {int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
internal override IArrowArrayStream NewReader<T>(T statement, Schema schema) => new HiveServer2Reader(statement, schema, dataTypeConversion: statement.Connection.DataTypeConversion);
@@ -141,10 +140,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port, ImpalaParameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port, ImpalaParameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue = GetAuthenticationHeaderValue(authTypeValue, username, password);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler = HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
@@ -164,24 +163,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback = (request, certificate, chain, policyErrors) =>
- {
-
- if (policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) && !TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned)) return false;
- if (policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) && !TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch)) return false;
-
- return true;
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue? GetAuthenticationHeaderValue(ImpalaAuthType authType, string? username, string? password)
{
if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) && (authType == ImpalaAuthType.Empty || authType == ImpalaAuthType.Basic))
diff --git a/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs b/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
index 43179b625..99ac368be 100644
--- a/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
+++ b/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
@@ -96,7 +96,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(ImpalaParameters.DataTypeConv, out string? dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
Properties.TryGetValue(ImpalaParameters.TLSOptions, out string? tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
}
protected override TTransport CreateTransport()
diff --git a/csharp/src/Drivers/Apache/Spark/README.md b/csharp/src/Drivers/Apache/Spark/README.md
index 1510c0291..5299a21e8 100644
--- a/csharp/src/Drivers/Apache/Spark/README.md
+++ b/csharp/src/Drivers/Apache/Spark/README.md
@@ -36,11 +36,15 @@ but can also be passed in the call to `AdbcDatabase.Connect`.
| `username` | The user name used for basic authentication | |
| `password` | The password for the user name used for basic authentication. | |
| `adbc.spark.data_type_conv` | Comma-separated list of data conversion options. Each option indicates the type of conversion to perform on data returned from the Spark server. <br><br>Allowed values: `none`, `scalar`. <br><br>Option `none` indicates there is no conversion from Spark type to native type (i.e., no conversion from String to Timestamp for Apache Spark over HTTP). Example `adbc.spark.conv_data_type=none`. <br><br>Option `scalar` will perform conversion (if necessary) from th [...]
-| `adbc.spark.tls_options` | Comma-separated list of TLS/SSL options. Each option indicates the TLS/SSL option when connecting to a Spark server. <br><br>Allowed values: `allow_self_signed`, `allow_hostname_mismatch`. <br><br>Option `allow_self_signed` allows certificate errors due to an unknown certificate authority, typically when using a self-signed certificate. Option `allow_hostname_mismatch` allow certificate errors due to a mismatch of the hostname. (e.g., when connecting through [...]
| `adbc.spark.connect_timeout_ms` | Sets the timeout (in milliseconds) to open a new session. Values can be 0 (infinite) or greater than zero. | `30000` |
| `adbc.apache.statement.batch_size` | Sets the maximum number of rows to retrieve in a single batch request. | `50000` |
| `adbc.apache.statement.polltime_ms` | If polling is necessary to get a result, this option sets the length of time (in milliseconds) to wait between polls. | `500` |
| `adbc.apache.statement.query_timeout_s` | Sets the maximum time (in seconds) for a query to complete. Values can be 0 (infinite) or greater than zero. | `60` |
+| `adbc.http_options.tls.enabled` | If tls needs to enabled or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.disable_server_certificate_validation` | If tls/ssl server certificate validation needs to enabled or not. One of `True`, `False`. If set to True, all certificate validation errors are ignored | `False` |
+| `adbc.http_options.tls.allow_self_signed` | If self signed tls/ssl certificate needs to be allowed or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.allow_hostname_mismatch` | If hostname mismatch is allowed for ssl. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.trusted_certificate_path` | The full path of the tls/ssl certificate .pem file containing custom CA certificates for verifying the server when connecting over TLS | `` |
## Timeout Configuration
diff --git a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
index a586f4b55..e28ab4632 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
@@ -121,7 +121,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
Properties.TryGetValue(SparkParameters.Path, out string? path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port, SparkParameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port, SparkParameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -129,8 +129,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
{
Properties.TryGetValue(SparkParameters.DataTypeConv, out string? dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(SparkParameters.TLSOptions, out string? tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(SparkParameters.ConnectTimeoutMilliseconds, out string? connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -138,6 +136,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
? connectTimeoutMsValue
: throw new ArgumentOutOfRangeException(SparkParameters.ConnectTimeoutMilliseconds, connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 .. {int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
internal override IArrowArrayStream NewReader<T>(T statement, Schema schema) => new HiveServer2Reader(statement, schema, dataTypeConversion: statement.Connection.DataTypeConversion);
@@ -159,10 +158,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port, SparkParameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port, SparkParameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue = GetAuthenticationHeaderValue(authTypeValue, token, username, password, access_token);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler = HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
@@ -182,24 +181,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback = (request, certificate, chain, policyErrors) =>
- {
- if (policyErrors == SslPolicyErrors.None) return true;
-
- return
- (!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) || TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned))
- && (!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) || TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch));
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue? GetAuthenticationHeaderValue(SparkAuthType authType, string? token, string? username, string? password, string? access_token)
{
if (!string.IsNullOrEmpty(token) && (authType == SparkAuthType.Empty || authType == SparkAuthType.Token))
diff --git a/csharp/src/Drivers/Apache/Spark/SparkParameters.cs b/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
index d3bd9be6d..8e75ae3f5 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
@@ -32,7 +32,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
public const string AuthType = "adbc.spark.auth_type";
public const string Type = "adbc.spark.type";
public const string DataTypeConv = "adbc.spark.data_type_conv";
- public const string TLSOptions = "adbc.spark.tls_options";
public const string ConnectTimeoutMilliseconds = "adbc.spark.connect_timeout_ms";
}
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs b/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
index 992e5ffb1..b481080bc 100644
--- a/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
@@ -34,16 +34,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
Assert.Throws(exceptionType, () => DataTypeConversionParser.Parse(dataTypeConversion));
}
- [SkippableTheory]
- [MemberData(nameof(GetParametersTlsOptionTestData))]
- internal void TestParametersTlsOptionParse(string? tlsOptions, HiveServer2TlsOption expected, Type? exceptionType = default)
- {
- if (exceptionType == default)
- Assert.Equal(expected, TlsOptionsParser.Parse(tlsOptions));
- else
- Assert.Throws(exceptionType, () => TlsOptionsParser.Parse(tlsOptions));
- }
-
public static IEnumerable<object?[]> GetParametersDataTypeConvTestData()
{
// Default
@@ -68,26 +58,5 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
yield return new object?[] { $"xxx", DataTypeConversion.Empty, typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $"none,scalar,xxx", DataTypeConversion.None | DataTypeConversion.Scalar, typeof(ArgumentOutOfRangeException) };
}
-
- public static IEnumerable<object?[]> GetParametersTlsOptionTestData()
- {
- // Default
- yield return new object?[] { null, HiveServer2TlsOption.Empty };
- yield return new object?[] { "", HiveServer2TlsOption.Empty};
- yield return new object?[] { " ", HiveServer2TlsOption.Empty };
- // Explicit
- yield return new object?[] { $"{TlsOptions.AllowSelfSigned}", HiveServer2TlsOption.AllowSelfSigned };
- yield return new object?[] { $"{TlsOptions.AllowHostnameMismatch}", HiveServer2TlsOption.AllowHostnameMismatch };
- // Ignore empty
- yield return new object?[] { $",{TlsOptions.AllowSelfSigned}", HiveServer2TlsOption.AllowSelfSigned };
- yield return new object?[] { $",{TlsOptions.AllowHostnameMismatch},", HiveServer2TlsOption.AllowHostnameMismatch };
- // Combined, embedded space, mixed-case
- yield return new object?[] { $"{TlsOptions.AllowSelfSigned},{TlsOptions.AllowHostnameMismatch}", HiveServer2TlsOption.AllowSelfSigned | HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] { $"{TlsOptions.AllowHostnameMismatch},{TlsOptions.AllowSelfSigned}", HiveServer2TlsOption.AllowSelfSigned | HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] { $" {TlsOptions.AllowHostnameMismatch} , {TlsOptions.AllowSelfSigned} ", HiveServer2TlsOption.AllowSelfSigned | HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] { $"{TlsOptions.AllowSelfSigned.ToUpperInvariant()},{TlsOptions.AllowHostnameMismatch.ToUpperInvariant()}", HiveServer2TlsOption.AllowSelfSigned | HiveServer2TlsOption.AllowHostnameMismatch };
- // Invalid
- yield return new object?[] { $"xxx,{TlsOptions.AllowSelfSigned.ToUpperInvariant()},{TlsOptions.AllowHostnameMismatch.ToUpperInvariant()}", HiveServer2TlsOption.Empty, typeof(ArgumentOutOfRangeException) };
- }
}
}
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
index 79829ae40..c442a8a7e 100644
--- a/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
@@ -90,10 +90,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
{
parameters.Add(HiveServer2Parameters.DataTypeConv, testConfiguration.DataTypeConversion!);
}
- if (!string.IsNullOrEmpty(testConfiguration.TlsOptions))
- {
- parameters.Add(HiveServer2Parameters.TLSOptions, testConfiguration.TlsOptions!);
- }
if (!string.IsNullOrEmpty(testConfiguration.BatchSize))
{
parameters.Add(ApacheParameters.BatchSize, testConfiguration.BatchSize!);
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs b/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs
new file mode 100644
index 000000000..70d5534e6
--- /dev/null
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs
@@ -0,0 +1,59 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
+using Xunit;
+
+namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
+{
+ public class HiveServer2TlsImplTest
+ {
+ [SkippableTheory]
+ [MemberData(nameof(GetSslOptionsTestData))]
+ internal void TestValidateTlsOptions(Dictionary<string, string>? dataTypeConversion, TlsProperties expected, Type? exceptionType = default)
+ {
+ if (exceptionType == default)
+ Assert.Equivalent(expected, HiveServer2TlsImpl.GetHttpTlsOptions(dataTypeConversion ?? new Dictionary<string, string>()));
+ else
+ Assert.Throws(exceptionType, () => HiveServer2TlsImpl.GetHttpTlsOptions(dataTypeConversion ?? new Dictionary<string, string>()));
+ }
+
+ public static IEnumerable<object?[]> GetSslOptionsTestData()
+ {
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "False" } }, new TlsProperties { IsTlsEnabled = false } };
+ yield return new object?[] { new Dictionary<string, string> { { AdbcOptions.Uri, "https://arrow.apache.org" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false } };
+ // uri takes precedence over ssl option
+ yield return new object?[] { new Dictionary<string, string> { { AdbcOptions.Uri, "https://arrow.apache.org" }, { HttpTlsOptions.IsTlsEnabled, "False" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false } };
+ // other ssl options are ignored if disableServerCertificateValidation is set to true
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "True" }, { HttpTlsOptions.DisableServerCertificateValidation, "True" }, { HttpTlsOptions.AllowSelfSigned, "True" }, { HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = true } };
+ // other ssl options are ignored if ssl is disabled
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "False" }, { HttpTlsOptions.AllowSelfSigned, "True" }, { HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties { IsTlsEnabled = false } };
+ // case insensitive boolean string parsing
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "false" } }, new TlsProperties { IsTlsEnabled = false } };
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "True" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false, AllowSelfSigned = false, AllowHostnameMismatch = false } };
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "tRUe" }, { HttpTlsOptions.AllowSelfSigned, "true" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false, AllowSelfSigned = true, AllowHostnameMismatch = false } };
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "TruE" }, { HttpTlsOptions.AllowSelfSigned, "True" }, { HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false, AllowSelfSigned = true, AllowHostnameMismatch = true } };
+ // certificate path is ignored if self signed is not allowed
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "True" }, { HttpTlsOptions.AllowSelfSigned, "False" }, { HttpTlsOptions.AllowHostnameMismatch, "True" }, { HttpTlsOptions.TrustedCertificatePath, "" } }, new TlsProperties { IsTlsEnabled = true, DisableServerCertificateValidation = false, AllowSelfSigned = false, AllowHostnameMismatch = true } };
+ // invalid certificate path
+ yield return new object?[] { new Dictionary<string, string> { { HttpTlsOptions.IsTlsEnabled, "True" }, { HttpTlsOptions.AllowSelfSigned, "True" }, { HttpTlsOptions.AllowHostnameMismatch, "True" }, { HttpTlsOptions.TrustedCertificatePath, "" } }, null, typeof(FileNotFoundException) };
+ }
+ }
+}
diff --git a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
index 9c9124d6d..265ac6b96 100644
--- a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
+++ b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
@@ -101,10 +101,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark
{
parameters.Add(SparkParameters.DataTypeConv, testConfiguration.DataTypeConversion!);
}
- if (!string.IsNullOrEmpty(testConfiguration.TlsOptions))
- {
- parameters.Add(SparkParameters.TLSOptions, testConfiguration.TlsOptions!);
- }
if (!string.IsNullOrEmpty(testConfiguration.BatchSize))
{
parameters.Add(ApacheParameters.BatchSize, testConfiguration.BatchSize!);