You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2025/08/21 20:41:21 UTC
(arrow-adbc) branch main updated: feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)
This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new a90bb2de8 feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)
a90bb2de8 is described below
commit a90bb2de8efa80a16436c0b268366f13e919f444
Author: eric-wang-1990 <11...@users.noreply.github.com>
AuthorDate: Thu Aug 21 13:41:16 2025 -0700
feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)
# Add Environment Variable Configuration Support for Databricks Driver
## Overview
Introduces robust configuration loading from JSON files via environment
variables with intelligent property merging and configurable precedence
control.
## Usage
In the system variable create a new record with name
"DATABRICKS_CONFIG_FILE" and point to a local json file which hold the
driver config properties. The driver config file will be loaded
alongside with PowerBI.
## Key Features
### 🔧 Environment Variable Configuration
- **Default environment variable**: `DATABRICKS_CONFIG_FILE`
automatically loaded in constructor
- **Standard ADBC parameter**: The keys are standard adbc parameter:
https://github.com/apache/arrow-adbc/blob/main/csharp/src/Drivers/Databricks/readme.md
```json
{
"adbc.databricks.driver_config_take_precedence": "true",
"adbc.databricks.enable_pk_fk": "false"
}
```
### 🎛️ Flexible Property Merging
- **Automatic merging**: Environment config + constructor properties
- **Configurable precedence**: New
`adbc.databricks.driver_config_take_precedence` parameter
- **Smart defaults**: Constructor properties override environment config
by default
## Testing
- Test with PowerBI Desktop
- Added new unit test
## Backward Compatibility
- ✅ All existing code works unchanged
- ✅ Original constructor behavior preserved
- ✅ Same exception patterns maintained
- ✅ Constructor properties still take precedence by default
---
.../Drivers/Databricks/DatabricksConfiguration.cs | 148 ++++++++
.../src/Drivers/Databricks/DatabricksConnection.cs | 103 +++++-
.../src/Drivers/Databricks/DatabricksParameters.cs | 10 +
.../Databricks/Unit/DatabricksConfigurationTest.cs | 393 +++++++++++++++++++++
4 files changed, 653 insertions(+), 1 deletion(-)
diff --git a/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs b/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs
new file mode 100644
index 000000000..79bef2bbe
--- /dev/null
+++ b/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs
@@ -0,0 +1,148 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks
+{
+ /// <summary>
+ /// Configuration class for Databricks connection properties loaded from JSON files.
+ /// </summary>
+ public class DatabricksConfiguration
+ {
+ /// <summary>
+ /// Dictionary of connection properties.
+ /// </summary>
+ public Dictionary<string, string> Properties { get; set; } = new();
+
+ /// <summary>
+ /// Creates a DatabricksConnection from this configuration.
+ /// </summary>
+ /// <returns>A configured DatabricksConnection instance.</returns>
+ internal DatabricksConnection CreateConnection()
+ {
+ return new DatabricksConnection(Properties);
+ }
+
+ /// <summary>
+ /// Loads configuration from an environment variable that points to a JSON file.
+ /// </summary>
+ /// <param name="environmentVariable">Name of the environment variable containing the file path.</param>
+ /// <returns>DatabricksConfiguration loaded from the file.</returns>
+ /// <exception cref="InvalidOperationException">Thrown when environment variable is not set or file doesn't exist.</exception>
+ public static DatabricksConfiguration FromEnvironmentVariable(string environmentVariable)
+ {
+ if (!CanLoadFromEnvironment(environmentVariable, out string? filePath))
+ {
+ throw new InvalidOperationException($"Cannot load Databricks configuration from environment variable '{environmentVariable}'. Make sure the environment variable is set and points to a valid JSON file.");
+ }
+
+ return FromFile(filePath!);
+ }
+
+ /// <summary>
+ /// Loads configuration from a JSON file.
+ /// Supports free-form JSON structure with key-value pairs.
+ /// </summary>
+ /// <param name="filePath">Path to the JSON configuration file.</param>
+ /// <returns>DatabricksConfiguration loaded from the file.</returns>
+ /// <exception cref="FileNotFoundException">Thrown when the file doesn't exist.</exception>
+ /// <exception cref="JsonException">Thrown when the JSON is invalid.</exception>
+ public static DatabricksConfiguration FromFile(string filePath)
+ {
+ if (!File.Exists(filePath))
+ {
+ throw new FileNotFoundException($"Configuration file not found: {filePath}");
+ }
+
+ try
+ {
+ string json = File.ReadAllText(filePath);
+ var options = new JsonSerializerOptions
+ {
+ PropertyNameCaseInsensitive = true,
+ ReadCommentHandling = JsonCommentHandling.Skip,
+ AllowTrailingCommas = true
+ };
+
+ // Deserialize as flat dictionary (free-form JSON)
+ var properties = JsonSerializer.Deserialize<Dictionary<string, string>>(json, options);
+ if (properties == null)
+ {
+ throw new InvalidOperationException($"Failed to deserialize configuration from {filePath}");
+ }
+
+ return new DatabricksConfiguration { Properties = properties };
+ }
+ catch (JsonException ex)
+ {
+ throw new JsonException($"Invalid JSON in configuration file {filePath}: {ex.Message}", ex);
+ }
+ }
+
+ /// <summary>
+ /// Checks if a configuration can be loaded from an environment variable.
+ /// </summary>
+ /// <param name="environmentVariable">Name of the environment variable.</param>
+ /// <param name="filePath">Output parameter containing the file path if available.</param>
+ /// <returns>True if the configuration can be loaded, false otherwise.</returns>
+ public static bool CanLoadFromEnvironment(string environmentVariable, out string? filePath)
+ {
+ filePath = null;
+
+ if (string.IsNullOrWhiteSpace(environmentVariable))
+ {
+ return false;
+ }
+
+ filePath = Environment.GetEnvironmentVariable(environmentVariable);
+
+ if (string.IsNullOrWhiteSpace(filePath))
+ {
+ return false;
+ }
+
+ return File.Exists(filePath);
+ }
+
+ /// <summary>
+ /// Tries to load configuration from an environment variable.
+ /// Returns null if the environment variable is not set or file doesn't exist.
+ /// </summary>
+ /// <param name="environmentVariable">Name of the environment variable.</param>
+ /// <returns>DatabricksConfiguration if successful, null otherwise.</returns>
+ public static DatabricksConfiguration? TryFromEnvironmentVariable(string environmentVariable)
+ {
+ try
+ {
+ if (CanLoadFromEnvironment(environmentVariable, out string? filePath))
+ {
+ return FromFile(filePath!);
+ }
+ }
+ catch
+ {
+ // Suppress exceptions in try method
+ }
+
+ return null;
+ }
+ }
+}
diff --git a/csharp/src/Drivers/Databricks/DatabricksConnection.cs b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
index 9ffe7b657..d1595b95c 100644
--- a/csharp/src/Drivers/Databricks/DatabricksConnection.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
@@ -18,9 +18,11 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
+using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
+using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Drivers.Apache;
@@ -40,6 +42,11 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
internal static new readonly string s_assemblyName = ApacheUtility.GetAssemblyName(typeof(DatabricksConnection));
internal static new readonly string s_assemblyVersion = ApacheUtility.GetAssemblyVersion(typeof(DatabricksConnection));
+ /// <summary>
+ /// The environment variable name that contains the path to the default Databricks configuration file.
+ /// </summary>
+ public const string DefaultConfigEnvironmentVariable = "DATABRICKS_CONFIG_FILE";
+
private bool _applySSPWithQueries = false;
private bool _enableDirectResults = true;
private bool _enableMultipleCatalogSupport = true;
@@ -75,7 +82,7 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
private HttpClient? _authHttpClient;
- public DatabricksConnection(IReadOnlyDictionary<string, string> properties) : base(properties)
+ public DatabricksConnection(IReadOnlyDictionary<string, string> properties) : base(MergeWithDefaultEnvironmentConfig(properties))
{
ValidateProperties();
}
@@ -94,6 +101,100 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
return new ThreadSafeClient(new TCLIService.Client(protocol));
}
+ /// <summary>
+ /// Automatically merges properties from the default DATABRICKS_CONFIG_FILE environment variable with passed-in properties.
+ /// The merge priority is controlled by the "adbc.databricks.driver_config_take_precedence" property.
+ /// If DATABRICKS_CONFIG_FILE is not set or invalid, only passed-in properties are used.
+ /// </summary>
+ /// <param name="properties">Properties passed to constructor.</param>
+ /// <returns>Merged properties dictionary.</returns>
+ private static IReadOnlyDictionary<string, string> MergeWithDefaultEnvironmentConfig(IReadOnlyDictionary<string, string> properties)
+ {
+ // Try to load configuration from the default environment variable
+ var environmentConfig = DatabricksConfiguration.TryFromEnvironmentVariable(DefaultConfigEnvironmentVariable);
+
+ if (environmentConfig != null)
+ {
+ // Determine precedence setting - check passed-in properties first, then environment config
+ bool driverConfigTakesPrecedence = DetermineDriverConfigPrecedence(properties, environmentConfig.Properties);
+
+ if (driverConfigTakesPrecedence)
+ {
+ // Environment config properties override passed-in properties
+ return MergeProperties(properties, environmentConfig.Properties);
+ }
+ else
+ {
+ // Passed-in properties override environment config properties (default behavior)
+ return MergeProperties(environmentConfig.Properties, properties);
+ }
+ }
+
+ // No environment config available, use only passed-in properties
+ return properties;
+ }
+
+ /// <summary>
+ /// Determines whether driver configuration should take precedence based on the precedence property.
+ /// Checks passed-in properties first, then environment properties, defaulting to false.
+ /// </summary>
+ /// <param name="passedInProperties">Properties passed to constructor.</param>
+ /// <param name="environmentProperties">Properties loaded from environment configuration.</param>
+ /// <returns>True if driver config should take precedence, false otherwise.</returns>
+ private static bool DetermineDriverConfigPrecedence(IReadOnlyDictionary<string, string> passedInProperties, IReadOnlyDictionary<string, string> environmentProperties)
+ {
+ // Priority 1: Check passed-in properties for precedence setting
+ if (passedInProperties.TryGetValue(DatabricksParameters.DriverConfigTakePrecedence, out string? passedInValue))
+ {
+ if (bool.TryParse(passedInValue, out bool passedInPrecedence))
+ {
+ return passedInPrecedence;
+ }
+ }
+
+ // Priority 2: Check environment config for precedence setting
+ if (environmentProperties.TryGetValue(DatabricksParameters.DriverConfigTakePrecedence, out string? environmentValue))
+ {
+ if (bool.TryParse(environmentValue, out bool environmentPrecedence))
+ {
+ return environmentPrecedence;
+ }
+ }
+
+ // Default: Passed-in properties override environment config (current behavior)
+ return false;
+ }
+
+ /// <summary>
+ /// Merges two property dictionaries, with additional properties taking precedence.
+ /// </summary>
+ /// <param name="baseProperties">Base properties dictionary.</param>
+ /// <param name="additionalProperties">Additional properties to merge. These take precedence over base properties.</param>
+ /// <returns>Merged properties dictionary.</returns>
+ private static IReadOnlyDictionary<string, string> MergeProperties(IReadOnlyDictionary<string, string> baseProperties, IReadOnlyDictionary<string, string>? additionalProperties)
+ {
+ if (additionalProperties == null || additionalProperties.Count == 0)
+ {
+ return baseProperties;
+ }
+
+ var merged = new Dictionary<string, string>();
+
+ // Add base properties first
+ foreach (var kvp in baseProperties)
+ {
+ merged[kvp.Key] = kvp.Value;
+ }
+
+ // Additional properties override base properties
+ foreach (var kvp in additionalProperties)
+ {
+ merged[kvp.Key] = kvp.Value;
+ }
+
+ return merged;
+ }
+
private void ValidateProperties()
{
if (Properties.TryGetValue(DatabricksParameters.EnablePKFK, out string? enablePKFKStr))
diff --git a/csharp/src/Drivers/Databricks/DatabricksParameters.cs b/csharp/src/Drivers/Databricks/DatabricksParameters.cs
index abb7cfd31..f27993b37 100644
--- a/csharp/src/Drivers/Databricks/DatabricksParameters.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksParameters.cs
@@ -218,6 +218,16 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
/// Default value is empty if not specified.
/// </summary>
public const string IdentityFederationClientId = "adbc.databricks.identity_federation_client_id";
+
+ /// <summary>
+ /// Controls whether driver configuration takes precedence over passed-in properties during configuration merging.
+ /// When "true": Environment/driver config properties override passed-in constructor properties.
+ /// When "false" (default): Passed-in constructor properties override environment/driver config properties.
+ /// This property can be set either in the environment configuration file or in passed-in properties.
+ /// When set in both places, the value in passed-in properties takes precedence.
+ /// Default value is false if not specified.
+ /// </summary>
+ public const string DriverConfigTakePrecedence = "adbc.databricks.driver_config_take_precedence";
}
/// <summary>
diff --git a/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs b/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs
new file mode 100644
index 000000000..54df00d5d
--- /dev/null
+++ b/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs
@@ -0,0 +1,393 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using Apache.Arrow.Adbc.Drivers.Apache.Spark;
+using Apache.Arrow.Adbc.Drivers.Databricks;
+using Xunit;
+
+namespace Apache.Arrow.Adbc.Tests.Drivers.Databricks
+{
+ /// <summary>
+ /// Unit tests for the DatabricksConfiguration class.
+ /// </summary>
+ public class DatabricksConfigurationTest : IDisposable
+ {
+ private readonly List<string> _tempFiles = new();
+
+ /// <summary>
+ /// Tests that FileNotFoundException is thrown for non-existent files.
+ /// </summary>
+ [Fact]
+ public void FromFile_FileNotFound_ThrowsFileNotFoundException()
+ {
+ // Arrange
+ var nonExistentFile = Path.Combine(Path.GetTempPath(), "nonexistent.json");
+
+ // Act & Assert
+ Assert.Throws<FileNotFoundException>(() => DatabricksConfiguration.FromFile(nonExistentFile));
+ }
+
+ /// <summary>
+ /// Tests that JsonException is thrown for invalid JSON.
+ /// </summary>
+ [Fact]
+ public void FromFile_InvalidJson_ThrowsJsonException()
+ {
+ // Arrange
+ var invalidJsonFile = CreateTempFile("invalid.json", "{ invalid json");
+
+ // Act & Assert
+ Assert.Throws<JsonException>(() => DatabricksConfiguration.FromFile(invalidJsonFile));
+ }
+
+ /// <summary>
+ /// Tests loading configuration from empty JSON object.
+ /// </summary>
+ [Fact]
+ public void FromFile_EmptyJson_ReturnsEmptyConfiguration()
+ {
+ // Arrange
+ var emptyJsonFile = CreateTempFile("empty.json", "{}");
+
+ // Act
+ var config = DatabricksConfiguration.FromFile(emptyJsonFile);
+
+ // Assert
+ Assert.NotNull(config);
+ Assert.NotNull(config.Properties);
+ Assert.Empty(config.Properties);
+ }
+
+ /// <summary>
+ /// Tests that JSON with comments is parsed correctly.
+ /// </summary>
+ [Fact]
+ public void FromFile_JsonWithComments_IgnoresComments()
+ {
+ // Arrange
+ var jsonWithComments = $@"{{
+ // This is a test Databricks configuration
+ ""{SparkParameters.HostName}"": ""test.databricks.com"",
+ /* Multi-line
+ comment for auth */
+ ""{SparkParameters.AuthType}"": ""Bearer"",
+ ""{DatabricksParameters.UseCloudFetch}"": ""true""
+ }}";
+ var configFile = CreateTempFile("with-comments.json", jsonWithComments);
+
+ // Act
+ var config = DatabricksConfiguration.FromFile(configFile);
+
+ // Assert
+ Assert.NotNull(config);
+ Assert.Equal(3, config.Properties.Count);
+ Assert.Equal("test.databricks.com", config.Properties[SparkParameters.HostName]);
+ Assert.Equal("Bearer", config.Properties[SparkParameters.AuthType]);
+ Assert.Equal("true", config.Properties[DatabricksParameters.UseCloudFetch]);
+ }
+
+ /// <summary>
+ /// Tests that JSON with trailing commas is parsed correctly.
+ /// </summary>
+ [Fact]
+ public void FromFile_JsonWithTrailingCommas_ParsesSuccessfully()
+ {
+ // Arrange
+ var jsonWithTrailingCommas = $@"{{
+ ""{SparkParameters.HostName}"": ""test.databricks.com"",
+ ""{DatabricksParameters.MaxBytesPerFile}"": ""10485760"",
+ }}";
+ var configFile = CreateTempFile("with-trailing-commas.json", jsonWithTrailingCommas);
+
+ // Act
+ var config = DatabricksConfiguration.FromFile(configFile);
+
+ // Assert
+ Assert.NotNull(config);
+ Assert.Equal(2, config.Properties.Count);
+ Assert.Equal("test.databricks.com", config.Properties[SparkParameters.HostName]);
+ Assert.Equal("10485760", config.Properties[DatabricksParameters.MaxBytesPerFile]);
+ }
+
+ /// <summary>
+ /// Tests case-insensitive key parsing.
+ /// </summary>
+ [Theory]
+ [InlineData("hostname", "test.databricks.com")]
+ [InlineData("HOSTNAME", "test.databricks.com")]
+ [InlineData("HostName", "test.databricks.com")]
+ [InlineData("authtype", "Bearer")]
+ [InlineData("AuthType", "Bearer")]
+ public void FromFile_CaseInsensitiveKeys_ParsesCorrectly(string keyName, string expectedValue)
+ {
+ // Arrange
+ var json = $@"{{ ""{keyName}"": ""{expectedValue}"" }}";
+ var configFile = CreateTempFile("case-test.json", json);
+
+ // Act
+ var config = DatabricksConfiguration.FromFile(configFile);
+
+ // Assert
+ Assert.NotNull(config);
+ Assert.Single(config.Properties);
+ Assert.Contains(config.Properties, kvp => kvp.Value == expectedValue);
+ }
+
+ /// <summary>
+ /// Tests loading configuration from environment variable.
+ /// </summary>
+ [Fact]
+ public void FromEnvironmentVariable_ValidPath_LoadsConfiguration()
+ {
+ // Arrange
+ var properties = new Dictionary<string, string>
+ {
+ [SparkParameters.HostName] = "env.databricks.com",
+ [SparkParameters.AuthType] = "Bearer",
+ [DatabricksParameters.EnableDirectResults] = "true"
+ };
+ var configFile = CreateTempJsonFile(properties);
+ var envVar = "TEST_ENV_VAR_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+ try
+ {
+ Environment.SetEnvironmentVariable(envVar, configFile);
+
+ // Act
+ var config = DatabricksConfiguration.FromEnvironmentVariable(envVar);
+
+ // Assert
+ Assert.NotNull(config);
+ Assert.Equal(properties.Count, config.Properties.Count);
+ Assert.Equal("env.databricks.com", config.Properties[SparkParameters.HostName]);
+ Assert.Equal("true", config.Properties[DatabricksParameters.EnableDirectResults]);
+ }
+ finally
+ {
+ Environment.SetEnvironmentVariable(envVar, null);
+ }
+ }
+
+
+ /// <summary>
+ /// Tests TryFromEnvironmentVariable with invalid configuration.
+ /// </summary>
+ [Fact]
+ public void TryFromEnvironmentVariable_InvalidConfig_ReturnsNull()
+ {
+ // Arrange
+ var nonExistentEnvVar = "NONEXISTENT_TRY_VAR_" + Guid.NewGuid().ToString("N");
+
+ // Act
+ var config = DatabricksConfiguration.TryFromEnvironmentVariable(nonExistentEnvVar);
+
+ // Assert
+ Assert.Null(config);
+ }
+
+ /// <summary>
+ /// Tests CanLoadFromEnvironment with valid environment variable.
+ /// </summary>
+ [Fact]
+ public void CanLoadFromEnvironment_ValidPath_ReturnsTrue()
+ {
+ // Arrange
+ var configFile = CreateTempJsonFile(new Dictionary<string, string> { ["test"] = "value" });
+ var envVar = "TEST_CAN_LOAD_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+ try
+ {
+ Environment.SetEnvironmentVariable(envVar, configFile);
+
+ // Act
+ var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVar, out string? filePath);
+
+ // Assert
+ Assert.True(canLoad);
+ Assert.Equal(configFile, filePath);
+ }
+ finally
+ {
+ Environment.SetEnvironmentVariable(envVar, null);
+ }
+ }
+
+ /// <summary>
+ /// Tests CanLoadFromEnvironment with invalid paths.
+ /// </summary>
+ [Theory]
+ [InlineData(null)]
+ [InlineData("")]
+ [InlineData(" ")]
+ public void CanLoadFromEnvironment_InvalidEnvVar_ReturnsFalse(string? envVarName)
+ {
+ // Act
+ var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVarName!, out string? filePath);
+
+ // Assert
+ Assert.False(canLoad);
+ Assert.Null(filePath);
+ }
+
+ /// <summary>
+ /// Tests CanLoadFromEnvironment with unset environment variable.
+ /// </summary>
+ [Fact]
+ public void CanLoadFromEnvironment_UnsetVar_ReturnsFalse()
+ {
+ // Arrange
+ var unsetEnvVar = "UNSET_ENV_VAR_" + Guid.NewGuid().ToString("N");
+
+ // Act
+ var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(unsetEnvVar, out string? filePath);
+
+ // Assert
+ Assert.False(canLoad);
+ Assert.Null(filePath);
+ }
+
+ /// <summary>
+ /// Tests CanLoadFromEnvironment with non-existent file.
+ /// </summary>
+ [Fact]
+ public void CanLoadFromEnvironment_NonExistentFile_ReturnsFalse()
+ {
+ // Arrange
+ var envVar = "TEST_NONEXISTENT_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+ var nonExistentFile = Path.Combine(Path.GetTempPath(), "nonexistent.json");
+
+ try
+ {
+ Environment.SetEnvironmentVariable(envVar, nonExistentFile);
+
+ // Act
+ var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVar, out string? filePath);
+
+ // Assert
+ Assert.False(canLoad);
+ Assert.Equal(nonExistentFile, filePath);
+ }
+ finally
+ {
+ Environment.SetEnvironmentVariable(envVar, null);
+ }
+ }
+
+ /// <summary>
+ /// Tests that property values are actually overridden based on precedence settings in DatabricksConnection.
+ /// Validates that the correct property values (EnablePKFK and UseCloudFetch) are used based on precedence.
+ /// </summary>
+ [Theory]
+ [InlineData("true")] // Environment takes precedence, should use environment values (false, false)
+ [InlineData("false")] // Constructor takes precedence, should use constructor values (true, true)
+ public void DatabricksConnection_PropertyOverridePrecedence_ValidatesCorrectValues(string precedenceValue)
+ {
+ // Arrange
+ var envProperties = new Dictionary<string, string>
+ {
+ [DatabricksParameters.DriverConfigTakePrecedence] = precedenceValue,
+ [DatabricksParameters.EnablePKFK] = "false",
+ [DatabricksParameters.UseCloudFetch] = "false"
+ };
+ var configFile = CreateTempJsonFile(envProperties);
+ var envVar = "TEST_PRECEDENCE_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+ var constructorProperties = new Dictionary<string, string>
+ {
+ [SparkParameters.HostName] = "test-host",
+ [SparkParameters.Token] = "test-token",
+ [DatabricksParameters.EnablePKFK] = "true",
+ [DatabricksParameters.UseCloudFetch] = "true",
+ [DatabricksParameters.CloudFetchMaxRetries] = "5"
+ };
+
+ try
+ {
+ Environment.SetEnvironmentVariable(DatabricksConnection.DefaultConfigEnvironmentVariable, configFile);
+
+ // Act
+ using var connection = new DatabricksConnection(constructorProperties);
+
+ // Assert - Validate that the correct property values are being used based on precedence
+ if (precedenceValue == "true")
+ {
+ // Environment takes precedence: should use environment values (false, false)
+ Assert.False(connection.EnablePKFK, "Environment precedence should use EnablePKFK=false from environment config");
+ Assert.False(connection.UseCloudFetch, "Environment precedence should use UseCloudFetch=false from environment config");
+ }
+ else
+ {
+ // Constructor takes precedence: should use constructor values (true, true)
+ Assert.True(connection.EnablePKFK, "Constructor precedence should use EnablePKFK=true from constructor properties");
+ Assert.True(connection.UseCloudFetch, "Constructor precedence should use UseCloudFetch=true from constructor properties");
+ }
+ }
+ finally
+ {
+ Environment.SetEnvironmentVariable(DatabricksConnection.DefaultConfigEnvironmentVariable, null);
+ }
+ }
+
+
+ #region Helper Methods
+
+ /// <summary>
+ /// Creates a temporary JSON file with the specified properties.
+ /// </summary>
+ private string CreateTempJsonFile(Dictionary<string, string> properties)
+ {
+ var json = JsonSerializer.Serialize(properties, new JsonSerializerOptions { WriteIndented = true });
+ return CreateTempFile("config.json", json);
+ }
+
+ /// <summary>
+ /// Creates a temporary file with the specified content.
+ /// </summary>
+ private string CreateTempFile(string fileName, string content)
+ {
+ var tempFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}_{fileName}");
+ File.WriteAllText(tempFile, content);
+ _tempFiles.Add(tempFile);
+ return tempFile;
+ }
+
+ /// <summary>
+ /// Cleans up temporary files.
+ /// </summary>
+ public void Dispose()
+ {
+ foreach (var tempFile in _tempFiles)
+ {
+ try
+ {
+ if (File.Exists(tempFile))
+ File.Delete(tempFile);
+ }
+ catch
+ {
+ // Ignore cleanup errors
+ }
+ }
+ }
+
+ #endregion
+ }
+}