You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2025/08/21 20:41:21 UTC

(arrow-adbc) branch main updated: feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)

This is an automated email from the ASF dual-hosted git repository.

curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git


The following commit(s) were added to refs/heads/main by this push:
     new a90bb2de8 feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)
a90bb2de8 is described below

commit a90bb2de8efa80a16436c0b268366f13e919f444
Author: eric-wang-1990 <11...@users.noreply.github.com>
AuthorDate: Thu Aug 21 13:41:16 2025 -0700

    feat(csharp/src/Drivers/Apache): add env variable config override for databricks (#3304)
    
    # Add Environment Variable Configuration Support for Databricks Driver
    
    ## Overview
    Introduces robust configuration loading from JSON files via environment
    variables with intelligent property merging and configurable precedence
    control.
    
    ## Usage
    In the system variable create a new record with name
    "DATABRICKS_CONFIG_FILE" and point to a local json file which hold the
    driver config properties. The driver config file will be loaded
    alongside with PowerBI.
    
    ## Key Features
    
    ### 🔧 Environment Variable Configuration
    - **Default environment variable**: `DATABRICKS_CONFIG_FILE`
    automatically loaded in constructor
    - **Standard ADBC parameter**: The keys are standard adbc parameter:
    https://github.com/apache/arrow-adbc/blob/main/csharp/src/Drivers/Databricks/readme.md
    ```json
    {
      "adbc.databricks.driver_config_take_precedence": "true",
      "adbc.databricks.enable_pk_fk": "false"
    }
    ```
    
    ### 🎛️ Flexible Property Merging
    - **Automatic merging**: Environment config + constructor properties
    - **Configurable precedence**: New
    `adbc.databricks.driver_config_take_precedence` parameter
    - **Smart defaults**: Constructor properties override environment config
    by default
    
    ## Testing
    - Test with PowerBI Desktop
    - Added new unit test
    
    ## Backward Compatibility
    - ✅ All existing code works unchanged
    - ✅ Original constructor behavior preserved
    - ✅ Same exception patterns maintained
    - ✅ Constructor properties still take precedence by default
---
 .../Drivers/Databricks/DatabricksConfiguration.cs  | 148 ++++++++
 .../src/Drivers/Databricks/DatabricksConnection.cs | 103 +++++-
 .../src/Drivers/Databricks/DatabricksParameters.cs |  10 +
 .../Databricks/Unit/DatabricksConfigurationTest.cs | 393 +++++++++++++++++++++
 4 files changed, 653 insertions(+), 1 deletion(-)

diff --git a/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs b/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs
new file mode 100644
index 000000000..79bef2bbe
--- /dev/null
+++ b/csharp/src/Drivers/Databricks/DatabricksConfiguration.cs
@@ -0,0 +1,148 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks
+{
+    /// <summary>
+    /// Configuration class for Databricks connection properties loaded from JSON files.
+    /// </summary>
+    public class DatabricksConfiguration
+    {
+        /// <summary>
+        /// Dictionary of connection properties.
+        /// </summary>
+        public Dictionary<string, string> Properties { get; set; } = new();
+
+        /// <summary>
+        /// Creates a DatabricksConnection from this configuration.
+        /// </summary>
+        /// <returns>A configured DatabricksConnection instance.</returns>
+        internal DatabricksConnection CreateConnection()
+        {
+            return new DatabricksConnection(Properties);
+        }
+
+        /// <summary>
+        /// Loads configuration from an environment variable that points to a JSON file.
+        /// </summary>
+        /// <param name="environmentVariable">Name of the environment variable containing the file path.</param>
+        /// <returns>DatabricksConfiguration loaded from the file.</returns>
+        /// <exception cref="InvalidOperationException">Thrown when environment variable is not set or file doesn't exist.</exception>
+        public static DatabricksConfiguration FromEnvironmentVariable(string environmentVariable)
+        {
+            if (!CanLoadFromEnvironment(environmentVariable, out string? filePath))
+            {
+                throw new InvalidOperationException($"Cannot load Databricks configuration from environment variable '{environmentVariable}'. Make sure the environment variable is set and points to a valid JSON file.");
+            }
+
+            return FromFile(filePath!);
+        }
+
+        /// <summary>
+        /// Loads configuration from a JSON file.
+        /// Supports free-form JSON structure with key-value pairs.
+        /// </summary>
+        /// <param name="filePath">Path to the JSON configuration file.</param>
+        /// <returns>DatabricksConfiguration loaded from the file.</returns>
+        /// <exception cref="FileNotFoundException">Thrown when the file doesn't exist.</exception>
+        /// <exception cref="JsonException">Thrown when the JSON is invalid.</exception>
+        public static DatabricksConfiguration FromFile(string filePath)
+        {
+            if (!File.Exists(filePath))
+            {
+                throw new FileNotFoundException($"Configuration file not found: {filePath}");
+            }
+
+            try
+            {
+                string json = File.ReadAllText(filePath);
+                var options = new JsonSerializerOptions
+                {
+                    PropertyNameCaseInsensitive = true,
+                    ReadCommentHandling = JsonCommentHandling.Skip,
+                    AllowTrailingCommas = true
+                };
+
+                // Deserialize as flat dictionary (free-form JSON)
+                var properties = JsonSerializer.Deserialize<Dictionary<string, string>>(json, options);
+                if (properties == null)
+                {
+                    throw new InvalidOperationException($"Failed to deserialize configuration from {filePath}");
+                }
+
+                return new DatabricksConfiguration { Properties = properties };
+            }
+            catch (JsonException ex)
+            {
+                throw new JsonException($"Invalid JSON in configuration file {filePath}: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Checks if a configuration can be loaded from an environment variable.
+        /// </summary>
+        /// <param name="environmentVariable">Name of the environment variable.</param>
+        /// <param name="filePath">Output parameter containing the file path if available.</param>
+        /// <returns>True if the configuration can be loaded, false otherwise.</returns>
+        public static bool CanLoadFromEnvironment(string environmentVariable, out string? filePath)
+        {
+            filePath = null;
+
+            if (string.IsNullOrWhiteSpace(environmentVariable))
+            {
+                return false;
+            }
+
+            filePath = Environment.GetEnvironmentVariable(environmentVariable);
+
+            if (string.IsNullOrWhiteSpace(filePath))
+            {
+                return false;
+            }
+
+            return File.Exists(filePath);
+        }
+
+        /// <summary>
+        /// Tries to load configuration from an environment variable.
+        /// Returns null if the environment variable is not set or file doesn't exist.
+        /// </summary>
+        /// <param name="environmentVariable">Name of the environment variable.</param>
+        /// <returns>DatabricksConfiguration if successful, null otherwise.</returns>
+        public static DatabricksConfiguration? TryFromEnvironmentVariable(string environmentVariable)
+        {
+            try
+            {
+                if (CanLoadFromEnvironment(environmentVariable, out string? filePath))
+                {
+                    return FromFile(filePath!);
+                }
+            }
+            catch
+            {
+                // Suppress exceptions in try method
+            }
+
+            return null;
+        }
+    }
+}
diff --git a/csharp/src/Drivers/Databricks/DatabricksConnection.cs b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
index 9ffe7b657..d1595b95c 100644
--- a/csharp/src/Drivers/Databricks/DatabricksConnection.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksConnection.cs
@@ -18,9 +18,11 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Apache.Arrow.Adbc.Drivers.Apache;
@@ -40,6 +42,11 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
         internal static new readonly string s_assemblyName = ApacheUtility.GetAssemblyName(typeof(DatabricksConnection));
         internal static new readonly string s_assemblyVersion = ApacheUtility.GetAssemblyVersion(typeof(DatabricksConnection));
 
+        /// <summary>
+        /// The environment variable name that contains the path to the default Databricks configuration file.
+        /// </summary>
+        public const string DefaultConfigEnvironmentVariable = "DATABRICKS_CONFIG_FILE";
+
         private bool _applySSPWithQueries = false;
         private bool _enableDirectResults = true;
         private bool _enableMultipleCatalogSupport = true;
@@ -75,7 +82,7 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
 
         private HttpClient? _authHttpClient;
 
-        public DatabricksConnection(IReadOnlyDictionary<string, string> properties) : base(properties)
+        public DatabricksConnection(IReadOnlyDictionary<string, string> properties) : base(MergeWithDefaultEnvironmentConfig(properties))
         {
             ValidateProperties();
         }
@@ -94,6 +101,100 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
             return new ThreadSafeClient(new TCLIService.Client(protocol));
         }
 
+        /// <summary>
+        /// Automatically merges properties from the default DATABRICKS_CONFIG_FILE environment variable with passed-in properties.
+        /// The merge priority is controlled by the "adbc.databricks.driver_config_take_precedence" property.
+        /// If DATABRICKS_CONFIG_FILE is not set or invalid, only passed-in properties are used.
+        /// </summary>
+        /// <param name="properties">Properties passed to constructor.</param>
+        /// <returns>Merged properties dictionary.</returns>
+        private static IReadOnlyDictionary<string, string> MergeWithDefaultEnvironmentConfig(IReadOnlyDictionary<string, string> properties)
+        {
+            // Try to load configuration from the default environment variable
+            var environmentConfig = DatabricksConfiguration.TryFromEnvironmentVariable(DefaultConfigEnvironmentVariable);
+
+            if (environmentConfig != null)
+            {
+                // Determine precedence setting - check passed-in properties first, then environment config
+                bool driverConfigTakesPrecedence = DetermineDriverConfigPrecedence(properties, environmentConfig.Properties);
+
+                if (driverConfigTakesPrecedence)
+                {
+                    // Environment config properties override passed-in properties
+                    return MergeProperties(properties, environmentConfig.Properties);
+                }
+                else
+                {
+                    // Passed-in properties override environment config properties (default behavior)
+                    return MergeProperties(environmentConfig.Properties, properties);
+                }
+            }
+
+            // No environment config available, use only passed-in properties
+            return properties;
+        }
+
+        /// <summary>
+        /// Determines whether driver configuration should take precedence based on the precedence property.
+        /// Checks passed-in properties first, then environment properties, defaulting to false.
+        /// </summary>
+        /// <param name="passedInProperties">Properties passed to constructor.</param>
+        /// <param name="environmentProperties">Properties loaded from environment configuration.</param>
+        /// <returns>True if driver config should take precedence, false otherwise.</returns>
+        private static bool DetermineDriverConfigPrecedence(IReadOnlyDictionary<string, string> passedInProperties, IReadOnlyDictionary<string, string> environmentProperties)
+        {
+            // Priority 1: Check passed-in properties for precedence setting
+            if (passedInProperties.TryGetValue(DatabricksParameters.DriverConfigTakePrecedence, out string? passedInValue))
+            {
+                if (bool.TryParse(passedInValue, out bool passedInPrecedence))
+                {
+                    return passedInPrecedence;
+                }
+            }
+
+            // Priority 2: Check environment config for precedence setting
+            if (environmentProperties.TryGetValue(DatabricksParameters.DriverConfigTakePrecedence, out string? environmentValue))
+            {
+                if (bool.TryParse(environmentValue, out bool environmentPrecedence))
+                {
+                    return environmentPrecedence;
+                }
+            }
+
+            // Default: Passed-in properties override environment config (current behavior)
+            return false;
+        }
+
+        /// <summary>
+        /// Merges two property dictionaries, with additional properties taking precedence.
+        /// </summary>
+        /// <param name="baseProperties">Base properties dictionary.</param>
+        /// <param name="additionalProperties">Additional properties to merge. These take precedence over base properties.</param>
+        /// <returns>Merged properties dictionary.</returns>
+        private static IReadOnlyDictionary<string, string> MergeProperties(IReadOnlyDictionary<string, string> baseProperties, IReadOnlyDictionary<string, string>? additionalProperties)
+        {
+            if (additionalProperties == null || additionalProperties.Count == 0)
+            {
+                return baseProperties;
+            }
+
+            var merged = new Dictionary<string, string>();
+
+            // Add base properties first
+            foreach (var kvp in baseProperties)
+            {
+                merged[kvp.Key] = kvp.Value;
+            }
+
+            // Additional properties override base properties
+            foreach (var kvp in additionalProperties)
+            {
+                merged[kvp.Key] = kvp.Value;
+            }
+
+            return merged;
+        }
+
         private void ValidateProperties()
         {
             if (Properties.TryGetValue(DatabricksParameters.EnablePKFK, out string? enablePKFKStr))
diff --git a/csharp/src/Drivers/Databricks/DatabricksParameters.cs b/csharp/src/Drivers/Databricks/DatabricksParameters.cs
index abb7cfd31..f27993b37 100644
--- a/csharp/src/Drivers/Databricks/DatabricksParameters.cs
+++ b/csharp/src/Drivers/Databricks/DatabricksParameters.cs
@@ -218,6 +218,16 @@ namespace Apache.Arrow.Adbc.Drivers.Databricks
         /// Default value is empty if not specified.
         /// </summary>
         public const string IdentityFederationClientId = "adbc.databricks.identity_federation_client_id";
+
+        /// <summary>
+        /// Controls whether driver configuration takes precedence over passed-in properties during configuration merging.
+        /// When "true": Environment/driver config properties override passed-in constructor properties.
+        /// When "false" (default): Passed-in constructor properties override environment/driver config properties.
+        /// This property can be set either in the environment configuration file or in passed-in properties.
+        /// When set in both places, the value in passed-in properties takes precedence.
+        /// Default value is false if not specified.
+        /// </summary>
+        public const string DriverConfigTakePrecedence = "adbc.databricks.driver_config_take_precedence";
     }
 
     /// <summary>
diff --git a/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs b/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs
new file mode 100644
index 000000000..54df00d5d
--- /dev/null
+++ b/csharp/test/Drivers/Databricks/Unit/DatabricksConfigurationTest.cs
@@ -0,0 +1,393 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using Apache.Arrow.Adbc.Drivers.Apache.Spark;
+using Apache.Arrow.Adbc.Drivers.Databricks;
+using Xunit;
+
+namespace Apache.Arrow.Adbc.Tests.Drivers.Databricks
+{
+    /// <summary>
+    /// Unit tests for the DatabricksConfiguration class.
+    /// </summary>
+    public class DatabricksConfigurationTest : IDisposable
+    {
+        private readonly List<string> _tempFiles = new();
+
+        /// <summary>
+        /// Tests that FileNotFoundException is thrown for non-existent files.
+        /// </summary>
+        [Fact]
+        public void FromFile_FileNotFound_ThrowsFileNotFoundException()
+        {
+            // Arrange
+            var nonExistentFile = Path.Combine(Path.GetTempPath(), "nonexistent.json");
+
+            // Act & Assert
+            Assert.Throws<FileNotFoundException>(() => DatabricksConfiguration.FromFile(nonExistentFile));
+        }
+
+        /// <summary>
+        /// Tests that JsonException is thrown for invalid JSON.
+        /// </summary>
+        [Fact]
+        public void FromFile_InvalidJson_ThrowsJsonException()
+        {
+            // Arrange
+            var invalidJsonFile = CreateTempFile("invalid.json", "{ invalid json");
+
+            // Act & Assert
+            Assert.Throws<JsonException>(() => DatabricksConfiguration.FromFile(invalidJsonFile));
+        }
+
+        /// <summary>
+        /// Tests loading configuration from empty JSON object.
+        /// </summary>
+        [Fact]
+        public void FromFile_EmptyJson_ReturnsEmptyConfiguration()
+        {
+            // Arrange
+            var emptyJsonFile = CreateTempFile("empty.json", "{}");
+
+            // Act
+            var config = DatabricksConfiguration.FromFile(emptyJsonFile);
+
+            // Assert
+            Assert.NotNull(config);
+            Assert.NotNull(config.Properties);
+            Assert.Empty(config.Properties);
+        }
+
+        /// <summary>
+        /// Tests that JSON with comments is parsed correctly.
+        /// </summary>
+        [Fact]
+        public void FromFile_JsonWithComments_IgnoresComments()
+        {
+            // Arrange
+            var jsonWithComments = $@"{{
+                // This is a test Databricks configuration
+                ""{SparkParameters.HostName}"": ""test.databricks.com"",
+                /* Multi-line
+                   comment for auth */
+                ""{SparkParameters.AuthType}"": ""Bearer"",
+                ""{DatabricksParameters.UseCloudFetch}"": ""true""
+            }}";
+            var configFile = CreateTempFile("with-comments.json", jsonWithComments);
+
+            // Act
+            var config = DatabricksConfiguration.FromFile(configFile);
+
+            // Assert
+            Assert.NotNull(config);
+            Assert.Equal(3, config.Properties.Count);
+            Assert.Equal("test.databricks.com", config.Properties[SparkParameters.HostName]);
+            Assert.Equal("Bearer", config.Properties[SparkParameters.AuthType]);
+            Assert.Equal("true", config.Properties[DatabricksParameters.UseCloudFetch]);
+        }
+
+        /// <summary>
+        /// Tests that JSON with trailing commas is parsed correctly.
+        /// </summary>
+        [Fact]
+        public void FromFile_JsonWithTrailingCommas_ParsesSuccessfully()
+        {
+            // Arrange
+            var jsonWithTrailingCommas = $@"{{
+                ""{SparkParameters.HostName}"": ""test.databricks.com"",
+                ""{DatabricksParameters.MaxBytesPerFile}"": ""10485760"",
+            }}";
+            var configFile = CreateTempFile("with-trailing-commas.json", jsonWithTrailingCommas);
+
+            // Act
+            var config = DatabricksConfiguration.FromFile(configFile);
+
+            // Assert
+            Assert.NotNull(config);
+            Assert.Equal(2, config.Properties.Count);
+            Assert.Equal("test.databricks.com", config.Properties[SparkParameters.HostName]);
+            Assert.Equal("10485760", config.Properties[DatabricksParameters.MaxBytesPerFile]);
+        }
+
+        /// <summary>
+        /// Tests case-insensitive key parsing.
+        /// </summary>
+        [Theory]
+        [InlineData("hostname", "test.databricks.com")]
+        [InlineData("HOSTNAME", "test.databricks.com")]
+        [InlineData("HostName", "test.databricks.com")]
+        [InlineData("authtype", "Bearer")]
+        [InlineData("AuthType", "Bearer")]
+        public void FromFile_CaseInsensitiveKeys_ParsesCorrectly(string keyName, string expectedValue)
+        {
+            // Arrange
+            var json = $@"{{ ""{keyName}"": ""{expectedValue}"" }}";
+            var configFile = CreateTempFile("case-test.json", json);
+
+            // Act
+            var config = DatabricksConfiguration.FromFile(configFile);
+
+            // Assert
+            Assert.NotNull(config);
+            Assert.Single(config.Properties);
+            Assert.Contains(config.Properties, kvp => kvp.Value == expectedValue);
+        }
+
+        /// <summary>
+        /// Tests loading configuration from environment variable.
+        /// </summary>
+        [Fact]
+        public void FromEnvironmentVariable_ValidPath_LoadsConfiguration()
+        {
+            // Arrange
+            var properties = new Dictionary<string, string>
+            {
+                [SparkParameters.HostName] = "env.databricks.com",
+                [SparkParameters.AuthType] = "Bearer",
+                [DatabricksParameters.EnableDirectResults] = "true"
+            };
+            var configFile = CreateTempJsonFile(properties);
+            var envVar = "TEST_ENV_VAR_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+            try
+            {
+                Environment.SetEnvironmentVariable(envVar, configFile);
+
+                // Act
+                var config = DatabricksConfiguration.FromEnvironmentVariable(envVar);
+
+                // Assert
+                Assert.NotNull(config);
+                Assert.Equal(properties.Count, config.Properties.Count);
+                Assert.Equal("env.databricks.com", config.Properties[SparkParameters.HostName]);
+                Assert.Equal("true", config.Properties[DatabricksParameters.EnableDirectResults]);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(envVar, null);
+            }
+        }
+
+
+        /// <summary>
+        /// Tests TryFromEnvironmentVariable with invalid configuration.
+        /// </summary>
+        [Fact]
+        public void TryFromEnvironmentVariable_InvalidConfig_ReturnsNull()
+        {
+            // Arrange
+            var nonExistentEnvVar = "NONEXISTENT_TRY_VAR_" + Guid.NewGuid().ToString("N");
+
+            // Act
+            var config = DatabricksConfiguration.TryFromEnvironmentVariable(nonExistentEnvVar);
+
+            // Assert
+            Assert.Null(config);
+        }
+
+        /// <summary>
+        /// Tests CanLoadFromEnvironment with valid environment variable.
+        /// </summary>
+        [Fact]
+        public void CanLoadFromEnvironment_ValidPath_ReturnsTrue()
+        {
+            // Arrange
+            var configFile = CreateTempJsonFile(new Dictionary<string, string> { ["test"] = "value" });
+            var envVar = "TEST_CAN_LOAD_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+            try
+            {
+                Environment.SetEnvironmentVariable(envVar, configFile);
+
+                // Act
+                var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVar, out string? filePath);
+
+                // Assert
+                Assert.True(canLoad);
+                Assert.Equal(configFile, filePath);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(envVar, null);
+            }
+        }
+
+        /// <summary>
+        /// Tests CanLoadFromEnvironment with invalid paths.
+        /// </summary>
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        [InlineData("   ")]
+        public void CanLoadFromEnvironment_InvalidEnvVar_ReturnsFalse(string? envVarName)
+        {
+            // Act
+            var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVarName!, out string? filePath);
+
+            // Assert
+            Assert.False(canLoad);
+            Assert.Null(filePath);
+        }
+
+        /// <summary>
+        /// Tests CanLoadFromEnvironment with unset environment variable.
+        /// </summary>
+        [Fact]
+        public void CanLoadFromEnvironment_UnsetVar_ReturnsFalse()
+        {
+            // Arrange
+            var unsetEnvVar = "UNSET_ENV_VAR_" + Guid.NewGuid().ToString("N");
+
+            // Act
+            var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(unsetEnvVar, out string? filePath);
+
+            // Assert
+            Assert.False(canLoad);
+            Assert.Null(filePath);
+        }
+
+        /// <summary>
+        /// Tests CanLoadFromEnvironment with non-existent file.
+        /// </summary>
+        [Fact]
+        public void CanLoadFromEnvironment_NonExistentFile_ReturnsFalse()
+        {
+            // Arrange
+            var envVar = "TEST_NONEXISTENT_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+            var nonExistentFile = Path.Combine(Path.GetTempPath(), "nonexistent.json");
+
+            try
+            {
+                Environment.SetEnvironmentVariable(envVar, nonExistentFile);
+
+                // Act
+                var canLoad = DatabricksConfiguration.CanLoadFromEnvironment(envVar, out string? filePath);
+
+                // Assert
+                Assert.False(canLoad);
+                Assert.Equal(nonExistentFile, filePath);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(envVar, null);
+            }
+        }
+
+        /// <summary>
+        /// Tests that property values are actually overridden based on precedence settings in DatabricksConnection.
+        /// Validates that the correct property values (EnablePKFK and UseCloudFetch) are used based on precedence.
+        /// </summary>
+        [Theory]
+        [InlineData("true")] // Environment takes precedence, should use environment values (false, false)
+        [InlineData("false")] // Constructor takes precedence, should use constructor values (true, true)
+        public void DatabricksConnection_PropertyOverridePrecedence_ValidatesCorrectValues(string precedenceValue)
+        {
+            // Arrange
+            var envProperties = new Dictionary<string, string>
+            {
+                [DatabricksParameters.DriverConfigTakePrecedence] = precedenceValue,
+                [DatabricksParameters.EnablePKFK] = "false",
+                [DatabricksParameters.UseCloudFetch] = "false"
+            };
+            var configFile = CreateTempJsonFile(envProperties);
+            var envVar = "TEST_PRECEDENCE_" + Guid.NewGuid().ToString("N").Substring(0, 8);
+
+            var constructorProperties = new Dictionary<string, string>
+            {
+                [SparkParameters.HostName] = "test-host",
+                [SparkParameters.Token] = "test-token",
+                [DatabricksParameters.EnablePKFK] = "true",
+                [DatabricksParameters.UseCloudFetch] = "true",
+                [DatabricksParameters.CloudFetchMaxRetries] = "5"
+            };
+
+            try
+            {
+                Environment.SetEnvironmentVariable(DatabricksConnection.DefaultConfigEnvironmentVariable, configFile);
+
+                // Act
+                using var connection = new DatabricksConnection(constructorProperties);
+
+                // Assert - Validate that the correct property values are being used based on precedence
+                if (precedenceValue == "true")
+                {
+                    // Environment takes precedence: should use environment values (false, false)
+                    Assert.False(connection.EnablePKFK, "Environment precedence should use EnablePKFK=false from environment config");
+                    Assert.False(connection.UseCloudFetch, "Environment precedence should use UseCloudFetch=false from environment config");
+                }
+                else
+                {
+                    // Constructor takes precedence: should use constructor values (true, true)
+                    Assert.True(connection.EnablePKFK, "Constructor precedence should use EnablePKFK=true from constructor properties");
+                    Assert.True(connection.UseCloudFetch, "Constructor precedence should use UseCloudFetch=true from constructor properties");
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(DatabricksConnection.DefaultConfigEnvironmentVariable, null);
+            }
+        }
+
+
+        #region Helper Methods
+
+        /// <summary>
+        /// Creates a temporary JSON file with the specified properties.
+        /// </summary>
+        private string CreateTempJsonFile(Dictionary<string, string> properties)
+        {
+            var json = JsonSerializer.Serialize(properties, new JsonSerializerOptions { WriteIndented = true });
+            return CreateTempFile("config.json", json);
+        }
+
+        /// <summary>
+        /// Creates a temporary file with the specified content.
+        /// </summary>
+        private string CreateTempFile(string fileName, string content)
+        {
+            var tempFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}_{fileName}");
+            File.WriteAllText(tempFile, content);
+            _tempFiles.Add(tempFile);
+            return tempFile;
+        }
+
+        /// <summary>
+        /// Cleans up temporary files.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var tempFile in _tempFiles)
+            {
+                try
+                {
+                    if (File.Exists(tempFile))
+                        File.Delete(tempFile);
+                }
+                catch
+                {
+                    // Ignore cleanup errors
+                }
+            }
+        }
+
+        #endregion
+    }
+}